2024-11-26 15:24:48,398 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-26 15:24:48,410 main DEBUG Took 0.009603 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 15:24:48,410 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 15:24:48,410 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 15:24:48,411 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 15:24:48,412 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:24:48,420 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 15:24:48,435 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:24:48,436 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:24:48,437 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:24:48,438 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:24:48,439 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:24:48,439 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:24:48,440 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:24:48,441 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:24:48,442 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:24:48,442 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:24:48,443 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:24:48,443 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:24:48,444 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:24:48,444 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:24:48,445 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:24:48,446 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:24:48,446 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:24:48,447 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:24:48,447 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:24:48,448 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:24:48,448 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:24:48,449 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:24:48,449 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:24:48,450 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:24:48,450 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:24:48,451 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 15:24:48,453 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:24:48,455 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 15:24:48,458 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 15:24:48,458 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 15:24:48,460 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 15:24:48,461 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 15:24:48,471 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 15:24:48,474 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 15:24:48,476 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 15:24:48,476 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 15:24:48,477 main DEBUG createAppenders(={Console}) 2024-11-26 15:24:48,477 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-26 15:24:48,478 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-26 15:24:48,478 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-26 15:24:48,479 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 15:24:48,479 main DEBUG OutputStream closed 2024-11-26 15:24:48,480 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 15:24:48,480 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 15:24:48,480 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-26 15:24:48,577 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 15:24:48,580 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 15:24:48,582 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 15:24:48,583 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 15:24:48,584 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 15:24:48,585 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 15:24:48,585 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 15:24:48,585 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 15:24:48,586 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 15:24:48,586 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 15:24:48,587 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 15:24:48,587 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 15:24:48,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 15:24:48,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 15:24:48,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 15:24:48,589 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 15:24:48,589 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 15:24:48,590 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 15:24:48,593 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 15:24:48,594 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-26 15:24:48,594 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 15:24:48,595 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-26T15:24:48,921 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc 2024-11-26 15:24:48,926 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 15:24:48,926 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T15:24:48,935 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-26T15:24:48,964 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T15:24:48,989 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/cluster_1ef45550-199a-e5e8-727d-dd604c50436f, deleteOnExit=true 2024-11-26T15:24:48,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T15:24:48,991 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/test.cache.data in system properties and HBase conf 2024-11-26T15:24:48,992 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T15:24:48,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/hadoop.log.dir in system properties and HBase conf 2024-11-26T15:24:48,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T15:24:48,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T15:24:48,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T15:24:49,123 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T15:24:49,258 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T15:24:49,263 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T15:24:49,264 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T15:24:49,265 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T15:24:49,266 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T15:24:49,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T15:24:49,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T15:24:49,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T15:24:49,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T15:24:49,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T15:24:49,270 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/nfs.dump.dir in system properties and HBase conf 2024-11-26T15:24:49,271 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/java.io.tmpdir in system properties and HBase conf 2024-11-26T15:24:49,271 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T15:24:49,272 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T15:24:49,273 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T15:24:50,690 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-26T15:24:50,801 INFO [Time-limited test {}] log.Log(170): Logging initialized @3383ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T15:24:50,884 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T15:24:50,948 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T15:24:50,972 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T15:24:50,973 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T15:24:50,974 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T15:24:50,985 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T15:24:50,989 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/hadoop.log.dir/,AVAILABLE} 2024-11-26T15:24:50,990 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T15:24:51,185 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/java.io.tmpdir/jetty-localhost-46689-hadoop-hdfs-3_4_1-tests_jar-_-any-10838069502973993658/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T15:24:51,194 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:46689} 2024-11-26T15:24:51,194 INFO [Time-limited test {}] server.Server(415): Started @3777ms 2024-11-26T15:24:53,534 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T15:24:53,545 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T15:24:53,553 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T15:24:53,553 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T15:24:53,554 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T15:24:53,557 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6aa6ac55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/hadoop.log.dir/,AVAILABLE} 2024-11-26T15:24:53,558 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58f8c8ee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T15:24:53,704 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d1d0948{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/java.io.tmpdir/jetty-localhost-35761-hadoop-hdfs-3_4_1-tests_jar-_-any-1904696431485938275/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T15:24:53,705 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@49b49063{HTTP/1.1, (http/1.1)}{localhost:35761} 2024-11-26T15:24:53,706 INFO [Time-limited test {}] server.Server(415): Started @6289ms 2024-11-26T15:24:53,766 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T15:24:55,098 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/cluster_1ef45550-199a-e5e8-727d-dd604c50436f/data/data2/current/BP-1143902897-172.17.0.3-1732634689990/current, will proceed with Du for space computation calculation, 2024-11-26T15:24:55,098 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/cluster_1ef45550-199a-e5e8-727d-dd604c50436f/data/data1/current/BP-1143902897-172.17.0.3-1732634689990/current, will proceed with Du for space computation calculation, 2024-11-26T15:24:55,169 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T15:24:55,239 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf177d506bfb5e2e2 with lease ID 0xb046593f5ccb4608: Processing first storage report for DS-028ca4f5-5c67-43fd-adab-d2157daf4182 from datanode DatanodeRegistration(127.0.0.1:40785, datanodeUuid=257274e4-7b31-4604-b9d8-d69272e12d3c, infoPort=36211, infoSecurePort=0, ipcPort=39397, storageInfo=lv=-57;cid=testClusterID;nsid=273108602;c=1732634689990) 2024-11-26T15:24:55,240 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf177d506bfb5e2e2 with lease ID 0xb046593f5ccb4608: from storage DS-028ca4f5-5c67-43fd-adab-d2157daf4182 node DatanodeRegistration(127.0.0.1:40785, datanodeUuid=257274e4-7b31-4604-b9d8-d69272e12d3c, infoPort=36211, infoSecurePort=0, ipcPort=39397, storageInfo=lv=-57;cid=testClusterID;nsid=273108602;c=1732634689990), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-26T15:24:55,241 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf177d506bfb5e2e2 with lease ID 0xb046593f5ccb4608: Processing first storage report for DS-5a922c8f-88d2-4ffa-ae4a-42d022933623 from datanode DatanodeRegistration(127.0.0.1:40785, datanodeUuid=257274e4-7b31-4604-b9d8-d69272e12d3c, infoPort=36211, infoSecurePort=0, ipcPort=39397, storageInfo=lv=-57;cid=testClusterID;nsid=273108602;c=1732634689990) 2024-11-26T15:24:55,241 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc 2024-11-26T15:24:55,241 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf177d506bfb5e2e2 with lease ID 0xb046593f5ccb4608: from storage DS-5a922c8f-88d2-4ffa-ae4a-42d022933623 node DatanodeRegistration(127.0.0.1:40785, datanodeUuid=257274e4-7b31-4604-b9d8-d69272e12d3c, infoPort=36211, infoSecurePort=0, ipcPort=39397, storageInfo=lv=-57;cid=testClusterID;nsid=273108602;c=1732634689990), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T15:24:55,354 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/cluster_1ef45550-199a-e5e8-727d-dd604c50436f/zookeeper_0, clientPort=51747, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/cluster_1ef45550-199a-e5e8-727d-dd604c50436f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/cluster_1ef45550-199a-e5e8-727d-dd604c50436f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T15:24:55,370 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51747 2024-11-26T15:24:55,386 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:24:55,391 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:24:55,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741825_1001 (size=7) 2024-11-26T15:24:56,137 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6 with version=8 2024-11-26T15:24:56,138 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/hbase-staging 2024-11-26T15:24:56,233 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T15:24:56,489 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6cc2bc8bf733:0 server-side Connection retries=45 2024-11-26T15:24:56,498 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T15:24:56,499 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T15:24:56,505 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T15:24:56,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T15:24:56,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T15:24:56,668 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T15:24:56,723 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T15:24:56,735 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T15:24:56,740 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T15:24:56,772 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 31829 (auto-detected) 2024-11-26T15:24:56,773 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-26T15:24:56,797 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37081 2024-11-26T15:24:56,821 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37081 connecting to ZooKeeper ensemble=127.0.0.1:51747 2024-11-26T15:24:56,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:370810x0, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T15:24:56,925 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37081-0x10177bf52480000 connected 2024-11-26T15:24:57,018 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:24:57,020 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:24:57,036 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T15:24:57,040 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6, hbase.cluster.distributed=false 2024-11-26T15:24:57,066 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T15:24:57,070 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37081 2024-11-26T15:24:57,071 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37081 2024-11-26T15:24:57,071 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37081 2024-11-26T15:24:57,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37081 2024-11-26T15:24:57,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37081 2024-11-26T15:24:57,180 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6cc2bc8bf733:0 server-side Connection retries=45 2024-11-26T15:24:57,181 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T15:24:57,182 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T15:24:57,182 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T15:24:57,182 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T15:24:57,182 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T15:24:57,185 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T15:24:57,188 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T15:24:57,189 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45729 2024-11-26T15:24:57,191 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45729 connecting to ZooKeeper ensemble=127.0.0.1:51747 2024-11-26T15:24:57,192 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:24:57,196 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:24:57,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:457290x0, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T15:24:57,206 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:457290x0, quorum=127.0.0.1:51747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T15:24:57,206 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45729-0x10177bf52480001 connected 2024-11-26T15:24:57,210 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T15:24:57,218 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T15:24:57,221 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T15:24:57,227 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T15:24:57,228 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45729 2024-11-26T15:24:57,228 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45729 2024-11-26T15:24:57,228 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45729 2024-11-26T15:24:57,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45729 2024-11-26T15:24:57,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45729 2024-11-26T15:24:57,245 DEBUG [M:0;6cc2bc8bf733:37081 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6cc2bc8bf733:37081 2024-11-26T15:24:57,246 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6cc2bc8bf733,37081,1732634696319 2024-11-26T15:24:57,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T15:24:57,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T15:24:57,261 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6cc2bc8bf733,37081,1732634696319 2024-11-26T15:24:57,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T15:24:57,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:24:57,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:24:57,301 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T15:24:57,303 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6cc2bc8bf733,37081,1732634696319 from backup master directory 2024-11-26T15:24:57,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T15:24:57,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6cc2bc8bf733,37081,1732634696319 2024-11-26T15:24:57,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T15:24:57,312 WARN [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T15:24:57,312 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6cc2bc8bf733,37081,1732634696319 2024-11-26T15:24:57,314 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T15:24:57,316 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T15:24:57,377 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/hbase.id] with ID: abffd310-7d55-49f7-a58a-b264a94194b6 2024-11-26T15:24:57,377 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/.tmp/hbase.id 2024-11-26T15:24:57,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741826_1002 (size=42) 2024-11-26T15:24:57,389 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/.tmp/hbase.id]:[hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/hbase.id] 2024-11-26T15:24:57,441 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:24:57,446 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T15:24:57,465 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-26T15:24:57,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:24:57,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:24:57,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741827_1003 (size=196) 2024-11-26T15:24:57,513 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T15:24:57,517 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T15:24:57,538 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:24:57,544 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:24:57,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741828_1004 (size=1189) 2024-11-26T15:24:58,000 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store 2024-11-26T15:24:58,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741829_1005 (size=34) 2024-11-26T15:24:58,430 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T15:24:58,433 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:24:58,435 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T15:24:58,435 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T15:24:58,436 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T15:24:58,437 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T15:24:58,438 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T15:24:58,438 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T15:24:58,439 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732634698435Disabling compacts and flushes for region at 1732634698435Disabling writes for close at 1732634698438 (+3 ms)Writing region close event to WAL at 1732634698438Closed at 1732634698438 2024-11-26T15:24:58,442 WARN [master/6cc2bc8bf733:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/.initializing 2024-11-26T15:24:58,443 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/WALs/6cc2bc8bf733,37081,1732634696319 2024-11-26T15:24:58,453 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T15:24:58,474 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6cc2bc8bf733%2C37081%2C1732634696319, suffix=, logDir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/WALs/6cc2bc8bf733,37081,1732634696319, archiveDir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/oldWALs, maxLogs=10 2024-11-26T15:24:58,499 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/WALs/6cc2bc8bf733,37081,1732634696319/6cc2bc8bf733%2C37081%2C1732634696319.1732634698481, exclude list is [], retry=0 2024-11-26T15:24:58,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40785,DS-028ca4f5-5c67-43fd-adab-d2157daf4182,DISK] 2024-11-26T15:24:58,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-26T15:24:58,557 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/WALs/6cc2bc8bf733,37081,1732634696319/6cc2bc8bf733%2C37081%2C1732634696319.1732634698481 2024-11-26T15:24:58,559 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36211:36211)] 2024-11-26T15:24:58,560 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:24:58,560 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:24:58,564 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:24:58,565 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:24:58,604 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:24:58,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T15:24:58,637 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:24:58,640 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:24:58,641 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:24:58,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T15:24:58,644 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:24:58,646 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:24:58,646 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:24:58,650 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T15:24:58,650 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:24:58,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:24:58,652 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:24:58,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T15:24:58,656 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:24:58,658 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:24:58,659 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:24:58,662 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:24:58,664 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:24:58,669 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:24:58,670 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:24:58,675 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T15:24:58,679 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:24:58,685 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:24:58,687 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70252569, jitterRate=0.046844854950904846}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T15:24:58,694 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732634698578Initializing all the Stores at 1732634698581 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634698582 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634698582Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634698583 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634698583Cleaning up temporary data from old regions at 1732634698670 (+87 ms)Region opened successfully at 1732634698694 (+24 ms) 2024-11-26T15:24:58,695 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T15:24:58,732 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c56509, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6cc2bc8bf733/172.17.0.3:0 2024-11-26T15:24:58,772 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T15:24:58,783 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T15:24:58,783 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T15:24:58,786 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T15:24:58,787 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T15:24:58,792 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T15:24:58,792 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T15:24:58,818 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T15:24:58,827 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T15:24:58,868 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T15:24:58,871 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T15:24:58,873 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T15:24:58,879 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T15:24:58,881 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T15:24:58,885 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T15:24:58,898 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T15:24:58,901 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T15:24:58,910 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T15:24:58,930 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T15:24:58,942 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T15:24:58,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T15:24:58,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T15:24:58,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:24:58,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:24:58,956 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6cc2bc8bf733,37081,1732634696319, sessionid=0x10177bf52480000, setting cluster-up flag (Was=false) 2024-11-26T15:24:58,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:24:58,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:24:59,163 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T15:24:59,166 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6cc2bc8bf733,37081,1732634696319 2024-11-26T15:24:59,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:24:59,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:24:59,216 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T15:24:59,218 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6cc2bc8bf733,37081,1732634696319 2024-11-26T15:24:59,223 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T15:24:59,234 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(746): ClusterId : abffd310-7d55-49f7-a58a-b264a94194b6 2024-11-26T15:24:59,237 DEBUG [RS:0;6cc2bc8bf733:45729 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T15:24:59,249 DEBUG [RS:0;6cc2bc8bf733:45729 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T15:24:59,250 DEBUG [RS:0;6cc2bc8bf733:45729 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T15:24:59,259 DEBUG [RS:0;6cc2bc8bf733:45729 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T15:24:59,260 DEBUG [RS:0;6cc2bc8bf733:45729 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@532ddb9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6cc2bc8bf733/172.17.0.3:0 2024-11-26T15:24:59,274 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6cc2bc8bf733:45729 2024-11-26T15:24:59,277 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T15:24:59,278 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T15:24:59,278 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T15:24:59,280 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(2659): reportForDuty to master=6cc2bc8bf733,37081,1732634696319 with port=45729, startcode=1732634697146 2024-11-26T15:24:59,291 DEBUG [RS:0;6cc2bc8bf733:45729 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T15:24:59,299 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T15:24:59,310 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T15:24:59,320 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T15:24:59,320 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38693, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T15:24:59,325 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37081 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:24:59,330 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6cc2bc8bf733,37081,1732634696319 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T15:24:59,339 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6cc2bc8bf733:0, corePoolSize=5, maxPoolSize=5 2024-11-26T15:24:59,339 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6cc2bc8bf733:0, corePoolSize=5, maxPoolSize=5 2024-11-26T15:24:59,339 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6cc2bc8bf733:0, corePoolSize=5, maxPoolSize=5 2024-11-26T15:24:59,339 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6cc2bc8bf733:0, corePoolSize=5, maxPoolSize=5 2024-11-26T15:24:59,339 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6cc2bc8bf733:0, corePoolSize=10, maxPoolSize=10 2024-11-26T15:24:59,340 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:24:59,340 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6cc2bc8bf733:0, corePoolSize=2, maxPoolSize=2 2024-11-26T15:24:59,340 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:24:59,341 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732634729341 2024-11-26T15:24:59,343 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T15:24:59,344 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T15:24:59,346 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T15:24:59,347 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T15:24:59,347 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T15:24:59,348 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T15:24:59,348 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T15:24:59,348 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T15:24:59,349 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T15:24:59,354 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:24:59,354 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-26T15:24:59,354 WARN [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-26T15:24:59,354 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T15:24:59,355 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T15:24:59,356 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T15:24:59,356 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T15:24:59,359 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T15:24:59,359 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T15:24:59,361 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6cc2bc8bf733:0:becomeActiveMaster-HFileCleaner.large.0-1732634699360,5,FailOnTimeoutGroup] 2024-11-26T15:24:59,363 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6cc2bc8bf733:0:becomeActiveMaster-HFileCleaner.small.0-1732634699361,5,FailOnTimeoutGroup] 2024-11-26T15:24:59,363 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T15:24:59,363 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T15:24:59,364 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T15:24:59,365 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T15:24:59,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741831_1007 (size=1321) 2024-11-26T15:24:59,455 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(2659): reportForDuty to master=6cc2bc8bf733,37081,1732634696319 with port=45729, startcode=1732634697146 2024-11-26T15:24:59,458 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37081 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6cc2bc8bf733,45729,1732634697146 2024-11-26T15:24:59,459 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37081 {}] master.ServerManager(517): Registering regionserver=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:24:59,467 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6 2024-11-26T15:24:59,467 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44107 2024-11-26T15:24:59,467 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T15:24:59,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T15:24:59,501 DEBUG [RS:0;6cc2bc8bf733:45729 {}] zookeeper.ZKUtil(111): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6cc2bc8bf733,45729,1732634697146 2024-11-26T15:24:59,501 WARN [RS:0;6cc2bc8bf733:45729 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T15:24:59,501 INFO [RS:0;6cc2bc8bf733:45729 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:24:59,501 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/WALs/6cc2bc8bf733,45729,1732634697146 2024-11-26T15:24:59,503 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6cc2bc8bf733,45729,1732634697146] 2024-11-26T15:24:59,539 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T15:24:59,555 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T15:24:59,560 INFO [RS:0;6cc2bc8bf733:45729 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T15:24:59,560 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T15:24:59,561 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T15:24:59,566 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T15:24:59,567 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T15:24:59,568 DEBUG [RS:0;6cc2bc8bf733:45729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:24:59,568 DEBUG [RS:0;6cc2bc8bf733:45729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:24:59,568 DEBUG [RS:0;6cc2bc8bf733:45729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:24:59,568 DEBUG [RS:0;6cc2bc8bf733:45729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:24:59,569 DEBUG [RS:0;6cc2bc8bf733:45729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:24:59,569 DEBUG [RS:0;6cc2bc8bf733:45729 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0, corePoolSize=2, maxPoolSize=2 2024-11-26T15:24:59,569 DEBUG [RS:0;6cc2bc8bf733:45729 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:24:59,569 DEBUG [RS:0;6cc2bc8bf733:45729 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:24:59,569 DEBUG [RS:0;6cc2bc8bf733:45729 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:24:59,569 DEBUG [RS:0;6cc2bc8bf733:45729 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:24:59,569 DEBUG [RS:0;6cc2bc8bf733:45729 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:24:59,570 DEBUG [RS:0;6cc2bc8bf733:45729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:24:59,570 DEBUG [RS:0;6cc2bc8bf733:45729 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6cc2bc8bf733:0, corePoolSize=3, maxPoolSize=3 2024-11-26T15:24:59,570 DEBUG [RS:0;6cc2bc8bf733:45729 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0, corePoolSize=3, maxPoolSize=3 2024-11-26T15:24:59,571 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T15:24:59,571 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T15:24:59,571 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T15:24:59,571 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T15:24:59,571 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T15:24:59,571 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,45729,1732634697146-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T15:24:59,590 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T15:24:59,592 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,45729,1732634697146-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T15:24:59,592 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T15:24:59,592 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.Replication(171): 6cc2bc8bf733,45729,1732634697146 started 2024-11-26T15:24:59,610 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T15:24:59,610 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1482): Serving as 6cc2bc8bf733,45729,1732634697146, RpcServer on 6cc2bc8bf733/172.17.0.3:45729, sessionid=0x10177bf52480001 2024-11-26T15:24:59,611 DEBUG [RS:0;6cc2bc8bf733:45729 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T15:24:59,611 DEBUG [RS:0;6cc2bc8bf733:45729 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6cc2bc8bf733,45729,1732634697146 2024-11-26T15:24:59,611 DEBUG [RS:0;6cc2bc8bf733:45729 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6cc2bc8bf733,45729,1732634697146' 2024-11-26T15:24:59,611 DEBUG [RS:0;6cc2bc8bf733:45729 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T15:24:59,612 DEBUG [RS:0;6cc2bc8bf733:45729 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T15:24:59,613 DEBUG [RS:0;6cc2bc8bf733:45729 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T15:24:59,613 DEBUG [RS:0;6cc2bc8bf733:45729 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T15:24:59,613 DEBUG [RS:0;6cc2bc8bf733:45729 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6cc2bc8bf733,45729,1732634697146 2024-11-26T15:24:59,613 DEBUG [RS:0;6cc2bc8bf733:45729 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6cc2bc8bf733,45729,1732634697146' 2024-11-26T15:24:59,613 DEBUG [RS:0;6cc2bc8bf733:45729 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T15:24:59,614 DEBUG [RS:0;6cc2bc8bf733:45729 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T15:24:59,615 DEBUG [RS:0;6cc2bc8bf733:45729 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T15:24:59,615 INFO [RS:0;6cc2bc8bf733:45729 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T15:24:59,615 INFO [RS:0;6cc2bc8bf733:45729 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T15:24:59,719 INFO [RS:0;6cc2bc8bf733:45729 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T15:24:59,723 INFO [RS:0;6cc2bc8bf733:45729 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6cc2bc8bf733%2C45729%2C1732634697146, suffix=, logDir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/WALs/6cc2bc8bf733,45729,1732634697146, archiveDir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/oldWALs, maxLogs=32 2024-11-26T15:24:59,738 DEBUG [RS:0;6cc2bc8bf733:45729 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/WALs/6cc2bc8bf733,45729,1732634697146/6cc2bc8bf733%2C45729%2C1732634697146.1732634699725, exclude list is [], retry=0 2024-11-26T15:24:59,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40785,DS-028ca4f5-5c67-43fd-adab-d2157daf4182,DISK] 2024-11-26T15:24:59,746 INFO [RS:0;6cc2bc8bf733:45729 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/WALs/6cc2bc8bf733,45729,1732634697146/6cc2bc8bf733%2C45729%2C1732634697146.1732634699725 2024-11-26T15:24:59,747 DEBUG [RS:0;6cc2bc8bf733:45729 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36211:36211)] 2024-11-26T15:24:59,769 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T15:24:59,770 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6 2024-11-26T15:24:59,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741833_1009 (size=32) 2024-11-26T15:25:00,183 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:25:00,187 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T15:25:00,190 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T15:25:00,190 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:00,192 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:25:00,192 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T15:25:00,196 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T15:25:00,196 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:00,197 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:25:00,197 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T15:25:00,201 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T15:25:00,201 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:00,202 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:25:00,202 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T15:25:00,206 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T15:25:00,206 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:00,207 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:25:00,208 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T15:25:00,210 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740 2024-11-26T15:25:00,211 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740 2024-11-26T15:25:00,215 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T15:25:00,216 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T15:25:00,217 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T15:25:00,220 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T15:25:00,225 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:25:00,227 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60353765, jitterRate=-0.10065881907939911}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T15:25:00,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732634700183Initializing all the Stores at 1732634700186 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634700186Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634700186Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634700186Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634700186Cleaning up temporary data from old regions at 1732634700216 (+30 ms)Region opened successfully at 1732634700231 (+15 ms) 2024-11-26T15:25:00,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T15:25:00,231 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T15:25:00,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T15:25:00,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T15:25:00,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T15:25:00,233 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T15:25:00,234 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732634700231Disabling compacts and flushes for region at 1732634700231Disabling writes for close at 1732634700232 (+1 ms)Writing region close event to WAL at 1732634700233 (+1 ms)Closed at 1732634700233 2024-11-26T15:25:00,238 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T15:25:00,238 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T15:25:00,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T15:25:00,253 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T15:25:00,256 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T15:25:00,409 DEBUG [6cc2bc8bf733:37081 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T15:25:00,420 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:00,426 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6cc2bc8bf733,45729,1732634697146, state=OPENING 2024-11-26T15:25:00,489 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T15:25:00,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:25:00,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:25:00,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:25:00,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:25:00,503 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T15:25:00,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:25:00,679 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T15:25:00,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45319, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T15:25:00,696 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T15:25:00,696 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:25:00,696 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T15:25:00,700 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6cc2bc8bf733%2C45729%2C1732634697146.meta, suffix=.meta, logDir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/WALs/6cc2bc8bf733,45729,1732634697146, archiveDir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/oldWALs, maxLogs=32 2024-11-26T15:25:00,716 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/WALs/6cc2bc8bf733,45729,1732634697146/6cc2bc8bf733%2C45729%2C1732634697146.meta.1732634700702.meta, exclude list is [], retry=0 2024-11-26T15:25:00,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40785,DS-028ca4f5-5c67-43fd-adab-d2157daf4182,DISK] 2024-11-26T15:25:00,723 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/WALs/6cc2bc8bf733,45729,1732634697146/6cc2bc8bf733%2C45729%2C1732634697146.meta.1732634700702.meta 2024-11-26T15:25:00,724 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36211:36211)] 2024-11-26T15:25:00,724 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:25:00,726 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T15:25:00,728 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T15:25:00,734 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T15:25:00,738 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T15:25:00,738 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:25:00,738 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T15:25:00,739 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T15:25:00,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T15:25:00,743 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T15:25:00,744 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:00,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:25:00,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T15:25:00,747 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T15:25:00,747 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:00,748 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:25:00,748 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T15:25:00,750 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T15:25:00,750 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:00,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:25:00,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T15:25:00,752 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T15:25:00,753 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:00,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:25:00,754 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T15:25:00,755 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740 2024-11-26T15:25:00,758 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740 2024-11-26T15:25:00,761 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T15:25:00,761 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T15:25:00,762 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T15:25:00,765 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T15:25:00,767 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74395538, jitterRate=0.10857990384101868}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T15:25:00,767 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T15:25:00,769 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732634700739Writing region info on filesystem at 1732634700739Initializing all the Stores at 1732634700741 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634700741Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634700741Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634700742 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634700742Cleaning up temporary data from old regions at 1732634700761 (+19 ms)Running coprocessor post-open hooks at 1732634700767 (+6 ms)Region opened successfully at 1732634700769 (+2 ms) 2024-11-26T15:25:00,775 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732634700671 2024-11-26T15:25:00,787 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T15:25:00,788 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T15:25:00,790 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:00,792 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6cc2bc8bf733,45729,1732634697146, state=OPEN 2024-11-26T15:25:00,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T15:25:00,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T15:25:00,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:25:00,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:25:00,837 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:00,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T15:25:00,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6cc2bc8bf733,45729,1732634697146 in 333 msec 2024-11-26T15:25:00,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T15:25:00,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 602 msec 2024-11-26T15:25:00,858 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T15:25:00,858 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T15:25:00,877 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:00,878 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:00,896 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:00,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33543, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:00,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6700 sec 2024-11-26T15:25:00,926 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732634700926, completionTime=-1 2024-11-26T15:25:00,929 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-26T15:25:00,929 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T15:25:00,956 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-26T15:25:00,956 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732634760956 2024-11-26T15:25:00,956 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732634820956 2024-11-26T15:25:00,956 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-26T15:25:00,958 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,37081,1732634696319-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T15:25:00,959 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,37081,1732634696319-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T15:25:00,959 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,37081,1732634696319-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T15:25:00,960 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6cc2bc8bf733:37081, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T15:25:00,961 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T15:25:00,961 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T15:25:00,967 DEBUG [master/6cc2bc8bf733:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T15:25:00,992 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.679sec 2024-11-26T15:25:00,993 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T15:25:00,994 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T15:25:00,995 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T15:25:00,995 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T15:25:00,995 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T15:25:00,996 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,37081,1732634696319-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T15:25:00,997 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,37081,1732634696319-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T15:25:01,005 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T15:25:01,006 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T15:25:01,006 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,37081,1732634696319-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T15:25:01,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44068459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:01,051 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T15:25:01,051 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T15:25:01,055 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:01,058 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:01,066 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28163b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:01,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:01,075 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:01,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45edd1b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:01,079 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:01,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:01,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:01,123 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6cc2bc8bf733,37081,1732634696319 2024-11-26T15:25:01,143 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=710, ProcessCount=11, AvailableMemoryMB=6891 2024-11-26T15:25:01,156 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T15:25:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 6cc2bc8bf733,37081,1732634696319 2024-11-26T15:25:01,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@49bd28a3 2024-11-26T15:25:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T15:25:01,169 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T15:25:01,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T15:25:01,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T15:25:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:01,189 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T15:25:01,191 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-26T15:25:01,191 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:01,194 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T15:25:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T15:25:01,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741835_1011 (size=959) 2024-11-26T15:25:01,210 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6 2024-11-26T15:25:01,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741836_1012 (size=53) 2024-11-26T15:25:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T15:25:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T15:25:01,621 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:25:01,621 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4c75b81af65e8ac430a5b8a152e75175, disabling compactions & flushes 2024-11-26T15:25:01,621 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:01,621 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:01,621 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. after waiting 0 ms 2024-11-26T15:25:01,621 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:01,622 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:01,622 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4c75b81af65e8ac430a5b8a152e75175: Waiting for close lock at 1732634701621Disabling compacts and flushes for region at 1732634701621Disabling writes for close at 1732634701621Writing region close event to WAL at 1732634701622 (+1 ms)Closed at 1732634701622 2024-11-26T15:25:01,624 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T15:25:01,630 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732634701624"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732634701624"}]},"ts":"1732634701624"} 2024-11-26T15:25:01,635 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T15:25:01,637 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T15:25:01,640 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634701638"}]},"ts":"1732634701638"} 2024-11-26T15:25:01,644 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T15:25:01,646 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c75b81af65e8ac430a5b8a152e75175, ASSIGN}] 2024-11-26T15:25:01,649 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c75b81af65e8ac430a5b8a152e75175, ASSIGN 2024-11-26T15:25:01,652 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c75b81af65e8ac430a5b8a152e75175, ASSIGN; state=OFFLINE, location=6cc2bc8bf733,45729,1732634697146; forceNewPlan=false, retain=false 2024-11-26T15:25:01,804 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4c75b81af65e8ac430a5b8a152e75175, regionState=OPENING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:01,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c75b81af65e8ac430a5b8a152e75175, ASSIGN because future has completed 2024-11-26T15:25:01,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:25:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T15:25:01,980 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:01,981 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:25:01,981 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:01,982 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:25:01,982 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:01,982 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:01,985 INFO [StoreOpener-4c75b81af65e8ac430a5b8a152e75175-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:01,989 INFO [StoreOpener-4c75b81af65e8ac430a5b8a152e75175-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c75b81af65e8ac430a5b8a152e75175 columnFamilyName A 2024-11-26T15:25:01,989 DEBUG [StoreOpener-4c75b81af65e8ac430a5b8a152e75175-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:01,991 INFO [StoreOpener-4c75b81af65e8ac430a5b8a152e75175-1 {}] regionserver.HStore(327): Store=4c75b81af65e8ac430a5b8a152e75175/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:25:01,992 INFO [StoreOpener-4c75b81af65e8ac430a5b8a152e75175-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:01,995 INFO [StoreOpener-4c75b81af65e8ac430a5b8a152e75175-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c75b81af65e8ac430a5b8a152e75175 columnFamilyName B 2024-11-26T15:25:01,995 DEBUG [StoreOpener-4c75b81af65e8ac430a5b8a152e75175-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:01,997 INFO [StoreOpener-4c75b81af65e8ac430a5b8a152e75175-1 {}] regionserver.HStore(327): Store=4c75b81af65e8ac430a5b8a152e75175/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:25:01,997 INFO [StoreOpener-4c75b81af65e8ac430a5b8a152e75175-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:02,001 INFO [StoreOpener-4c75b81af65e8ac430a5b8a152e75175-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c75b81af65e8ac430a5b8a152e75175 columnFamilyName C 2024-11-26T15:25:02,001 DEBUG [StoreOpener-4c75b81af65e8ac430a5b8a152e75175-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:02,002 INFO [StoreOpener-4c75b81af65e8ac430a5b8a152e75175-1 {}] regionserver.HStore(327): Store=4c75b81af65e8ac430a5b8a152e75175/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:25:02,003 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:02,006 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:02,007 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:02,016 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:02,017 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:02,020 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T15:25:02,024 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:02,042 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:25:02,044 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4c75b81af65e8ac430a5b8a152e75175; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70164515, jitterRate=0.045532748103141785}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T15:25:02,044 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:02,046 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4c75b81af65e8ac430a5b8a152e75175: Running coprocessor pre-open hook at 1732634701982Writing region info on filesystem at 1732634701982Initializing all the Stores at 1732634701985 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634701985Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634701985Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634701985Cleaning up temporary data from old regions at 1732634702017 (+32 ms)Running coprocessor post-open hooks at 1732634702044 (+27 ms)Region opened successfully at 1732634702045 (+1 ms) 2024-11-26T15:25:02,049 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., pid=6, masterSystemTime=1732634701970 2024-11-26T15:25:02,055 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:02,056 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:02,059 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4c75b81af65e8ac430a5b8a152e75175, regionState=OPEN, openSeqNum=2, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:02,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:25:02,070 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37081 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=6cc2bc8bf733,45729,1732634697146, table=TestAcidGuarantees, region=4c75b81af65e8ac430a5b8a152e75175. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T15:25:02,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T15:25:02,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 in 270 msec 2024-11-26T15:25:02,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T15:25:02,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c75b81af65e8ac430a5b8a152e75175, ASSIGN in 445 msec 2024-11-26T15:25:02,102 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T15:25:02,103 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634702102"}]},"ts":"1732634702102"} 2024-11-26T15:25:02,108 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T15:25:02,110 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T15:25:02,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 931 msec 2024-11-26T15:25:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T15:25:02,334 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:02,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b7cf82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,341 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:02,341 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:02,353 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:02,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:02,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:02,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55f99e3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:02,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:02,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:02,363 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:02,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@595680d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,367 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:02,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:02,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:02,376 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:02,380 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T15:25:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 6cc2bc8bf733,37081,1732634696319 2024-11-26T15:25:02,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@684bb5df 2024-11-26T15:25:02,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T15:25:02,398 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T15:25:02,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c22fa37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:02,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:02,421 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:02,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7907fcac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:02,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:02,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:02,430 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:02,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73fc0926, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7b17c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,444 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:02,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:02,450 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:02,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:02,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:02,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33551cc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:02,454 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4474c69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6068bcec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:02,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:02,467 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@797de42a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:02,472 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59cb0f4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11fbf093, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,478 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:02,478 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:02,487 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51278417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:02,500 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:02,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@759c8e82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f1c2cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,505 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:02,505 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:02,507 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:02,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:02,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:02,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3050d1c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:02,513 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:02,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@288d5d04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eea3ba9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,529 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:02,529 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:02,532 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:02,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:02,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32637125, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:02,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:02,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:02,539 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:02,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bde52ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d6a39a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,545 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:02,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:02,547 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23451635, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:02,552 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:02,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@146b39e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b7950b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:02,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:02,573 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:02,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:02,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:02,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3635d5dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:02,577 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a9e52e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b18ef51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:02,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:02,582 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78890e56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:02,586 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2615693e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:02,614 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:02,617 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:02,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:02,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:02,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:25:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:25:02,636 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:02,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:02,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:02,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:02,645 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:02,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:25:02,650 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:02,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:02,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:02,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:02,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:02,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:25:02,656 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:02,656 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:02,657 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:02,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:02,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:02,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:25:02,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:02,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:25:02,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:25:02,674 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:02,676 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:02,677 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-26T15:25:02,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:25:02,686 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:25:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T15:25:02,692 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:25:02,694 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:25:02,726 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:02,737 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:02,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:02,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:02,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:25:02,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:02,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:02,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:25:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T15:25:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:02,830 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:02,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-26T15:25:02,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T15:25:02,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:02,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:02,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:02,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:02,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T15:25:03,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T15:25:03,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:03,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:03,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:03,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/3d8998b2ae1a4388ab95de90621b8418 is 50, key is test_row_0/A:col10/1732634702804/Put/seqid=0 2024-11-26T15:25:03,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741837_1013 (size=14341) 2024-11-26T15:25:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634763131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634763134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634763147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634763164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634763173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T15:25:03,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:03,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:03,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:03,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634763294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634763295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634763297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T15:25:03,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634763304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634763304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T15:25:03,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:03,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:03,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:03,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634763504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634763515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634763515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634763534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634763535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/3d8998b2ae1a4388ab95de90621b8418 2024-11-26T15:25:03,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T15:25:03,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:03,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:03,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:03,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T15:25:03,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:03,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:03,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:03,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/083ab5891bff46498c497c5cc58a40e8 is 50, key is test_row_0/B:col10/1732634702804/Put/seqid=0 2024-11-26T15:25:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T15:25:03,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741838_1014 (size=12001) 2024-11-26T15:25:03,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/083ab5891bff46498c497c5cc58a40e8 2024-11-26T15:25:03,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634763824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634763833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634763845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634763845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634763845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:03,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/8b203e0193e74666a2a309aeafbf5ebf is 50, key is test_row_0/C:col10/1732634702804/Put/seqid=0 2024-11-26T15:25:03,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T15:25:03,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:03,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:03,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:03,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:03,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741839_1015 (size=12001) 2024-11-26T15:25:03,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/8b203e0193e74666a2a309aeafbf5ebf 2024-11-26T15:25:03,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/3d8998b2ae1a4388ab95de90621b8418 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/3d8998b2ae1a4388ab95de90621b8418 2024-11-26T15:25:03,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/3d8998b2ae1a4388ab95de90621b8418, entries=200, sequenceid=14, filesize=14.0 K 2024-11-26T15:25:04,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/083ab5891bff46498c497c5cc58a40e8 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/083ab5891bff46498c497c5cc58a40e8 2024-11-26T15:25:04,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/083ab5891bff46498c497c5cc58a40e8, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T15:25:04,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/8b203e0193e74666a2a309aeafbf5ebf as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/8b203e0193e74666a2a309aeafbf5ebf 2024-11-26T15:25:04,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/8b203e0193e74666a2a309aeafbf5ebf, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T15:25:04,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T15:25:04,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:04,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:04,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:04,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:04,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:04,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 4c75b81af65e8ac430a5b8a152e75175 in 1230ms, sequenceid=14, compaction requested=false 2024-11-26T15:25:04,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:04,068 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T15:25:04,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:04,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T15:25:04,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:04,232 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T15:25:04,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/c5b27d88a9f14b4b90797b5bfdda865d is 50, key is test_row_0/A:col10/1732634703133/Put/seqid=0 2024-11-26T15:25:04,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:04,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741840_1016 (size=12001) 2024-11-26T15:25:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:04,352 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/c5b27d88a9f14b4b90797b5bfdda865d 2024-11-26T15:25:04,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/a55fdb4a53d74cab8526256a44ee5e86 is 50, key is test_row_0/B:col10/1732634703133/Put/seqid=0 2024-11-26T15:25:04,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:04,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634764403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634764402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:04,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:04,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741841_1017 (size=12001) 2024-11-26T15:25:04,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634764408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:04,434 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/a55fdb4a53d74cab8526256a44ee5e86 2024-11-26T15:25:04,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634764416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:04,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634764428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:04,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/191405d332c54c11a86a59150845f7c5 is 50, key is test_row_0/C:col10/1732634703133/Put/seqid=0 2024-11-26T15:25:04,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741842_1018 (size=12001) 2024-11-26T15:25:04,520 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/191405d332c54c11a86a59150845f7c5 2024-11-26T15:25:04,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/c5b27d88a9f14b4b90797b5bfdda865d as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/c5b27d88a9f14b4b90797b5bfdda865d 2024-11-26T15:25:04,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634764536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:04,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:04,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634764544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:04,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634764544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:04,558 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/c5b27d88a9f14b4b90797b5bfdda865d, entries=150, sequenceid=38, filesize=11.7 K 2024-11-26T15:25:04,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/a55fdb4a53d74cab8526256a44ee5e86 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/a55fdb4a53d74cab8526256a44ee5e86 2024-11-26T15:25:04,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:04,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634764554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:04,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:04,580 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/a55fdb4a53d74cab8526256a44ee5e86, entries=150, sequenceid=38, filesize=11.7 K 2024-11-26T15:25:04,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/191405d332c54c11a86a59150845f7c5 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/191405d332c54c11a86a59150845f7c5 2024-11-26T15:25:04,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634764586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:04,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:04,616 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/191405d332c54c11a86a59150845f7c5, entries=150, sequenceid=38, filesize=11.7 K 2024-11-26T15:25:04,618 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4c75b81af65e8ac430a5b8a152e75175 in 385ms, sequenceid=38, compaction requested=false 2024-11-26T15:25:04,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:04,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:04,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-26T15:25:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-26T15:25:04,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T15:25:04,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9290 sec 2024-11-26T15:25:04,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9930 sec 2024-11-26T15:25:04,772 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:04,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-26T15:25:04,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:04,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/e653a5e891394351b919c08f3a1140b0 is 50, key is test_row_0/A:col10/1732634704768/Put/seqid=0 2024-11-26T15:25:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T15:25:04,838 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:04,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:25:04,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-26T15:25:04,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T15:25:04,844 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:25:04,846 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:25:04,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:25:04,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741843_1019 (size=12001) 2024-11-26T15:25:04,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T15:25:05,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T15:25:05,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:05,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:05,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:05,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:05,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:05,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:05,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634764993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634764998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634765002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634765018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634765018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634765128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634765128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T15:25:05,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:05,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:05,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634765144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:05,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634765145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T15:25:05,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634765164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/e653a5e891394351b919c08f3a1140b0 2024-11-26T15:25:05,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/6eb36aa9de694d5296d386fe6cb372d9 is 50, key is test_row_0/B:col10/1732634704768/Put/seqid=0 2024-11-26T15:25:05,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T15:25:05,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:05,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:05,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:05,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:05,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:05,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634765343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634765343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741844_1020 (size=12001) 2024-11-26T15:25:05,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/6eb36aa9de694d5296d386fe6cb372d9 2024-11-26T15:25:05,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634765363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634765373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/0d3d01b3496e45f9b7c60ba0216bf498 is 50, key is test_row_0/C:col10/1732634704768/Put/seqid=0 2024-11-26T15:25:05,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634765392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741845_1021 (size=12001) 2024-11-26T15:25:05,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/0d3d01b3496e45f9b7c60ba0216bf498 2024-11-26T15:25:05,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/e653a5e891394351b919c08f3a1140b0 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/e653a5e891394351b919c08f3a1140b0 2024-11-26T15:25:05,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/e653a5e891394351b919c08f3a1140b0, entries=150, sequenceid=51, filesize=11.7 K 2024-11-26T15:25:05,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/6eb36aa9de694d5296d386fe6cb372d9 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/6eb36aa9de694d5296d386fe6cb372d9 2024-11-26T15:25:05,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/6eb36aa9de694d5296d386fe6cb372d9, entries=150, sequenceid=51, filesize=11.7 K 2024-11-26T15:25:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T15:25:05,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T15:25:05,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:05,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:05,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:05,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:05,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:05,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/0d3d01b3496e45f9b7c60ba0216bf498 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/0d3d01b3496e45f9b7c60ba0216bf498 2024-11-26T15:25:05,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/0d3d01b3496e45f9b7c60ba0216bf498, entries=150, sequenceid=51, filesize=11.7 K 2024-11-26T15:25:05,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 4c75b81af65e8ac430a5b8a152e75175 in 723ms, sequenceid=51, compaction requested=true 2024-11-26T15:25:05,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:05,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:05,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:05,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:05,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:05,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:05,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:05,525 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:05,527 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T15:25:05,529 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:05,534 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:05,536 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/B is initiating minor compaction (all files) 2024-11-26T15:25:05,537 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/B in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:05,537 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/083ab5891bff46498c497c5cc58a40e8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/a55fdb4a53d74cab8526256a44ee5e86, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/6eb36aa9de694d5296d386fe6cb372d9] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=35.2 K 2024-11-26T15:25:05,539 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:05,539 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/A is initiating minor compaction (all files) 2024-11-26T15:25:05,540 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/A in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:05,540 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/3d8998b2ae1a4388ab95de90621b8418, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/c5b27d88a9f14b4b90797b5bfdda865d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/e653a5e891394351b919c08f3a1140b0] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=37.4 K 2024-11-26T15:25:05,540 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 083ab5891bff46498c497c5cc58a40e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732634702804 2024-11-26T15:25:05,541 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d8998b2ae1a4388ab95de90621b8418, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732634702774 2024-11-26T15:25:05,541 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting a55fdb4a53d74cab8526256a44ee5e86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732634703133 2024-11-26T15:25:05,542 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5b27d88a9f14b4b90797b5bfdda865d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732634703133 2024-11-26T15:25:05,543 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting e653a5e891394351b919c08f3a1140b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732634704410 2024-11-26T15:25:05,543 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eb36aa9de694d5296d386fe6cb372d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732634704410 2024-11-26T15:25:05,601 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#A#compaction#9 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:05,602 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/8eea54fc38784873959d4d3683053897 is 50, key is test_row_0/A:col10/1732634704768/Put/seqid=0 2024-11-26T15:25:05,608 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#B#compaction#10 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:05,609 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/825d96e2c5dc46058375f8ae8a06e986 is 50, key is test_row_0/B:col10/1732634704768/Put/seqid=0 2024-11-26T15:25:05,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T15:25:05,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:05,629 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T15:25:05,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741846_1022 (size=12104) 2024-11-26T15:25:05,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741847_1023 (size=12104) 2024-11-26T15:25:05,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:05,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/b2981d99a06d487cacf5bb10f7a097de is 50, key is test_row_0/A:col10/1732634705012/Put/seqid=0 2024-11-26T15:25:05,702 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/825d96e2c5dc46058375f8ae8a06e986 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/825d96e2c5dc46058375f8ae8a06e986 2024-11-26T15:25:05,737 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/B of 4c75b81af65e8ac430a5b8a152e75175 into 825d96e2c5dc46058375f8ae8a06e986(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:05,737 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:05,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741848_1024 (size=12001) 2024-11-26T15:25:05,741 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/B, priority=13, startTime=1732634705522; duration=0sec 2024-11-26T15:25:05,741 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:05,742 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:B 2024-11-26T15:25:05,742 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:05,744 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:05,744 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/C is initiating minor compaction (all files) 2024-11-26T15:25:05,744 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/C in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:05,745 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/8b203e0193e74666a2a309aeafbf5ebf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/191405d332c54c11a86a59150845f7c5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/0d3d01b3496e45f9b7c60ba0216bf498] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=35.2 K 2024-11-26T15:25:05,746 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b203e0193e74666a2a309aeafbf5ebf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732634702804 2024-11-26T15:25:05,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634765722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,756 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 191405d332c54c11a86a59150845f7c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732634703133 2024-11-26T15:25:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634765728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,757 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d3d01b3496e45f9b7c60ba0216bf498, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732634704410 2024-11-26T15:25:05,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634765757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,802 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#C#compaction#12 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:05,803 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/9728c409170b4432bd0eabef273f1179 is 50, key is test_row_0/C:col10/1732634704768/Put/seqid=0 2024-11-26T15:25:05,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634765748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634765757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741849_1025 (size=12104) 2024-11-26T15:25:05,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634765863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634765871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634765913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634765923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634765923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:05,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:05,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T15:25:06,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634766097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:06,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634766101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:06,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:06,124 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/8eea54fc38784873959d4d3683053897 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/8eea54fc38784873959d4d3683053897 2024-11-26T15:25:06,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634766133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:06,150 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/b2981d99a06d487cacf5bb10f7a097de 2024-11-26T15:25:06,157 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/A of 4c75b81af65e8ac430a5b8a152e75175 into 8eea54fc38784873959d4d3683053897(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:06,157 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:06,157 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/A, priority=13, startTime=1732634705496; duration=0sec 2024-11-26T15:25:06,158 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:06,158 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:A 2024-11-26T15:25:06,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634766154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:06,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:06,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634766163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:06,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/b5a94c7f78c14dd49f3aa2d376b1d0ed is 50, key is test_row_0/B:col10/1732634705012/Put/seqid=0 2024-11-26T15:25:06,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741850_1026 (size=12001) 2024-11-26T15:25:06,263 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/b5a94c7f78c14dd49f3aa2d376b1d0ed 2024-11-26T15:25:06,267 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/9728c409170b4432bd0eabef273f1179 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/9728c409170b4432bd0eabef273f1179 2024-11-26T15:25:06,274 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T15:25:06,284 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/C of 4c75b81af65e8ac430a5b8a152e75175 into 9728c409170b4432bd0eabef273f1179(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:06,284 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:06,284 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/C, priority=13, startTime=1732634705523; duration=0sec 2024-11-26T15:25:06,285 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:06,285 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:C 2024-11-26T15:25:06,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/271e9e7937074d0b82550f121ac4a5b3 is 50, key is test_row_0/C:col10/1732634705012/Put/seqid=0 2024-11-26T15:25:06,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741851_1027 (size=12001) 2024-11-26T15:25:06,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634766413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:06,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634766423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:06,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634766453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:06,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:06,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634766475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:06,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:06,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634766483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:06,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:06,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:06,719 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T15:25:06,720 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T15:25:06,723 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T15:25:06,723 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T15:25:06,723 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-26T15:25:06,724 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-26T15:25:06,725 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T15:25:06,725 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-26T15:25:06,752 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/271e9e7937074d0b82550f121ac4a5b3 2024-11-26T15:25:06,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/b2981d99a06d487cacf5bb10f7a097de as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b2981d99a06d487cacf5bb10f7a097de 2024-11-26T15:25:06,780 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b2981d99a06d487cacf5bb10f7a097de, entries=150, sequenceid=75, filesize=11.7 K 2024-11-26T15:25:06,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/b5a94c7f78c14dd49f3aa2d376b1d0ed as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b5a94c7f78c14dd49f3aa2d376b1d0ed 2024-11-26T15:25:06,795 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b5a94c7f78c14dd49f3aa2d376b1d0ed, entries=150, sequenceid=75, filesize=11.7 K 2024-11-26T15:25:06,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/271e9e7937074d0b82550f121ac4a5b3 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/271e9e7937074d0b82550f121ac4a5b3 2024-11-26T15:25:06,822 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/271e9e7937074d0b82550f121ac4a5b3, entries=150, sequenceid=75, filesize=11.7 K 2024-11-26T15:25:06,825 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4c75b81af65e8ac430a5b8a152e75175 in 1196ms, sequenceid=75, compaction requested=false 2024-11-26T15:25:06,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:06,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:06,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-26T15:25:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-26T15:25:06,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-26T15:25:06,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9830 sec 2024-11-26T15:25:06,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.0010 sec 2024-11-26T15:25:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:06,941 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:06,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-26T15:25:06,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/b5934c714e3444729ef972285ce74aea is 50, key is test_row_0/A:col10/1732634706938/Put/seqid=0 2024-11-26T15:25:06,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741852_1028 (size=16681) 2024-11-26T15:25:06,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/b5934c714e3444729ef972285ce74aea 2024-11-26T15:25:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T15:25:06,995 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:06,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:25:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-26T15:25:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T15:25:07,004 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:25:07,007 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:25:07,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:25:07,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/342a87181cfa4768b8d05ff0d30db8f6 is 50, key is test_row_0/B:col10/1732634706938/Put/seqid=0 2024-11-26T15:25:07,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741853_1029 (size=12001) 2024-11-26T15:25:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T15:25:07,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634767115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634767115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634767124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634767133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634767137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T15:25:07,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:07,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:07,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:07,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:07,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:07,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634767242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634767243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634767243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634767252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634767252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T15:25:07,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T15:25:07,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:07,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:07,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:07,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:07,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:07,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/342a87181cfa4768b8d05ff0d30db8f6 2024-11-26T15:25:07,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634767453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634767453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634767456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634767463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634767463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/93346c1d3a8e4723a0678110416e9227 is 50, key is test_row_0/C:col10/1732634706938/Put/seqid=0 2024-11-26T15:25:07,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T15:25:07,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:07,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:07,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:07,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:07,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:07,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741854_1030 (size=12001) 2024-11-26T15:25:07,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/93346c1d3a8e4723a0678110416e9227 2024-11-26T15:25:07,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/b5934c714e3444729ef972285ce74aea as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b5934c714e3444729ef972285ce74aea 2024-11-26T15:25:07,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b5934c714e3444729ef972285ce74aea, entries=250, sequenceid=92, filesize=16.3 K 2024-11-26T15:25:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/342a87181cfa4768b8d05ff0d30db8f6 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/342a87181cfa4768b8d05ff0d30db8f6 2024-11-26T15:25:07,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/342a87181cfa4768b8d05ff0d30db8f6, entries=150, sequenceid=92, filesize=11.7 K 2024-11-26T15:25:07,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/93346c1d3a8e4723a0678110416e9227 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/93346c1d3a8e4723a0678110416e9227 2024-11-26T15:25:07,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/93346c1d3a8e4723a0678110416e9227, entries=150, sequenceid=92, filesize=11.7 K 2024-11-26T15:25:07,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 4c75b81af65e8ac430a5b8a152e75175 in 685ms, sequenceid=92, compaction requested=true 2024-11-26T15:25:07,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:07,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:07,628 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:07,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:07,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:07,628 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:07,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:07,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:07,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:07,630 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:07,630 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/B is initiating minor compaction (all files) 2024-11-26T15:25:07,630 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/B in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:07,631 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/825d96e2c5dc46058375f8ae8a06e986, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b5a94c7f78c14dd49f3aa2d376b1d0ed, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/342a87181cfa4768b8d05ff0d30db8f6] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=35.3 K 2024-11-26T15:25:07,631 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:07,631 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/A is initiating minor compaction (all files) 2024-11-26T15:25:07,631 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/A in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:07,632 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/8eea54fc38784873959d4d3683053897, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b2981d99a06d487cacf5bb10f7a097de, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b5934c714e3444729ef972285ce74aea] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=39.8 K 2024-11-26T15:25:07,632 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 825d96e2c5dc46058375f8ae8a06e986, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732634704410 2024-11-26T15:25:07,633 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eea54fc38784873959d4d3683053897, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732634704410 2024-11-26T15:25:07,633 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting b5a94c7f78c14dd49f3aa2d376b1d0ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732634704948 2024-11-26T15:25:07,634 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2981d99a06d487cacf5bb10f7a097de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732634704948 2024-11-26T15:25:07,634 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 342a87181cfa4768b8d05ff0d30db8f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732634705741 2024-11-26T15:25:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T15:25:07,635 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5934c714e3444729ef972285ce74aea, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732634705716 2024-11-26T15:25:07,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T15:25:07,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:07,637 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-26T15:25:07,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/44e26d299e4246689290db787269edb5 is 50, key is test_row_0/A:col10/1732634707128/Put/seqid=0 2024-11-26T15:25:07,668 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#B#compaction#19 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:07,669 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/7ce38b74b1174457baf875abd9ae6ad9 is 50, key is test_row_0/B:col10/1732634706938/Put/seqid=0 2024-11-26T15:25:07,678 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#A#compaction#20 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:07,679 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/cf367c1d6e1f4725873d1f65f4a1477b is 50, key is test_row_0/A:col10/1732634706938/Put/seqid=0 2024-11-26T15:25:07,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741855_1031 (size=12001) 2024-11-26T15:25:07,722 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/44e26d299e4246689290db787269edb5 2024-11-26T15:25:07,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741856_1032 (size=12207) 2024-11-26T15:25:07,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/9a0848a5a6be4862a23aeb689d263302 is 50, key is test_row_0/B:col10/1732634707128/Put/seqid=0 2024-11-26T15:25:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:07,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:07,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741857_1033 (size=12207) 2024-11-26T15:25:07,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741858_1034 (size=12001) 2024-11-26T15:25:07,797 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/9a0848a5a6be4862a23aeb689d263302 2024-11-26T15:25:07,805 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/cf367c1d6e1f4725873d1f65f4a1477b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/cf367c1d6e1f4725873d1f65f4a1477b 2024-11-26T15:25:07,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634767797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634767801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,821 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/A of 4c75b81af65e8ac430a5b8a152e75175 into cf367c1d6e1f4725873d1f65f4a1477b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:07,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634767802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634767803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,823 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:07,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,823 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/A, priority=13, startTime=1732634707628; duration=0sec 2024-11-26T15:25:07,824 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:07,824 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:A 2024-11-26T15:25:07,824 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:07,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634767813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,826 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:07,828 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/C is initiating minor compaction (all files) 2024-11-26T15:25:07,828 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/C in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:07,829 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/9728c409170b4432bd0eabef273f1179, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/271e9e7937074d0b82550f121ac4a5b3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/93346c1d3a8e4723a0678110416e9227] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=35.3 K 2024-11-26T15:25:07,830 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9728c409170b4432bd0eabef273f1179, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732634704410 2024-11-26T15:25:07,831 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 271e9e7937074d0b82550f121ac4a5b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732634704948 2024-11-26T15:25:07,832 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93346c1d3a8e4723a0678110416e9227, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732634705741 2024-11-26T15:25:07,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/41345d97520b41f98e431298e6efb117 is 50, key is test_row_0/C:col10/1732634707128/Put/seqid=0 2024-11-26T15:25:07,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741859_1035 (size=12001) 2024-11-26T15:25:07,868 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:07,870 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/bd5d1ff10db247de9fc49bc7ec5bcaf8 is 50, key is test_row_0/C:col10/1732634706938/Put/seqid=0 2024-11-26T15:25:07,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741860_1036 (size=12207) 2024-11-26T15:25:07,923 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/bd5d1ff10db247de9fc49bc7ec5bcaf8 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/bd5d1ff10db247de9fc49bc7ec5bcaf8 2024-11-26T15:25:07,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634767923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,940 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/C of 4c75b81af65e8ac430a5b8a152e75175 into bd5d1ff10db247de9fc49bc7ec5bcaf8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:07,940 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:07,940 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/C, priority=13, startTime=1732634707628; duration=0sec 2024-11-26T15:25:07,940 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:07,940 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:C 2024-11-26T15:25:07,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634767933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634767934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634767934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:07,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:07,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634767934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T15:25:08,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634768143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634768153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634768153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634768155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634768163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,197 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/7ce38b74b1174457baf875abd9ae6ad9 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/7ce38b74b1174457baf875abd9ae6ad9 2024-11-26T15:25:08,234 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/B of 4c75b81af65e8ac430a5b8a152e75175 into 7ce38b74b1174457baf875abd9ae6ad9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:08,234 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:08,234 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/B, priority=13, startTime=1732634707628; duration=0sec 2024-11-26T15:25:08,234 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:08,234 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:B 2024-11-26T15:25:08,255 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/41345d97520b41f98e431298e6efb117 2024-11-26T15:25:08,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/44e26d299e4246689290db787269edb5 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/44e26d299e4246689290db787269edb5 2024-11-26T15:25:08,290 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/44e26d299e4246689290db787269edb5, entries=150, sequenceid=115, filesize=11.7 K 2024-11-26T15:25:08,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/9a0848a5a6be4862a23aeb689d263302 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/9a0848a5a6be4862a23aeb689d263302 2024-11-26T15:25:08,307 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/9a0848a5a6be4862a23aeb689d263302, entries=150, sequenceid=115, filesize=11.7 K 2024-11-26T15:25:08,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/41345d97520b41f98e431298e6efb117 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/41345d97520b41f98e431298e6efb117 2024-11-26T15:25:08,326 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/41345d97520b41f98e431298e6efb117, entries=150, sequenceid=115, filesize=11.7 K 2024-11-26T15:25:08,328 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 4c75b81af65e8ac430a5b8a152e75175 in 691ms, sequenceid=115, compaction requested=false 2024-11-26T15:25:08,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:08,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:08,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-26T15:25:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-26T15:25:08,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-26T15:25:08,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3240 sec 2024-11-26T15:25:08,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3410 sec 2024-11-26T15:25:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:08,475 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:08,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-26T15:25:08,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/f2de7a7b5c0848109ffd5d0703b09815 is 50, key is test_row_0/A:col10/1732634708471/Put/seqid=0 2024-11-26T15:25:08,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741861_1037 (size=12101) 2024-11-26T15:25:08,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/f2de7a7b5c0848109ffd5d0703b09815 2024-11-26T15:25:08,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/b1b4943bab5a40debef771c593ca5bf1 is 50, key is test_row_0/B:col10/1732634708471/Put/seqid=0 2024-11-26T15:25:08,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634768520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634768520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634768523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634768523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634768528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741862_1038 (size=12101) 2024-11-26T15:25:08,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/b1b4943bab5a40debef771c593ca5bf1 2024-11-26T15:25:08,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/d7abe8e46905409c8489b83dd3247985 is 50, key is test_row_0/C:col10/1732634708471/Put/seqid=0 2024-11-26T15:25:08,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741863_1039 (size=12101) 2024-11-26T15:25:08,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/d7abe8e46905409c8489b83dd3247985 2024-11-26T15:25:08,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/f2de7a7b5c0848109ffd5d0703b09815 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/f2de7a7b5c0848109ffd5d0703b09815 2024-11-26T15:25:08,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/f2de7a7b5c0848109ffd5d0703b09815, entries=150, sequenceid=134, filesize=11.8 K 2024-11-26T15:25:08,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/b1b4943bab5a40debef771c593ca5bf1 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b1b4943bab5a40debef771c593ca5bf1 2024-11-26T15:25:08,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634768633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634768633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b1b4943bab5a40debef771c593ca5bf1, entries=150, sequenceid=134, filesize=11.8 K 2024-11-26T15:25:08,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634768633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634768633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/d7abe8e46905409c8489b83dd3247985 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/d7abe8e46905409c8489b83dd3247985 2024-11-26T15:25:08,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634768642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/d7abe8e46905409c8489b83dd3247985, entries=150, sequenceid=134, filesize=11.8 K 2024-11-26T15:25:08,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 4c75b81af65e8ac430a5b8a152e75175 in 182ms, sequenceid=134, compaction requested=true 2024-11-26T15:25:08,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:08,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:08,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:08,658 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:08,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:08,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:08,658 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:08,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:08,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:08,659 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:08,660 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/A is initiating minor compaction (all files) 2024-11-26T15:25:08,660 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/A in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:08,660 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/cf367c1d6e1f4725873d1f65f4a1477b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/44e26d299e4246689290db787269edb5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/f2de7a7b5c0848109ffd5d0703b09815] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=35.5 K 2024-11-26T15:25:08,664 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:08,664 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf367c1d6e1f4725873d1f65f4a1477b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732634705741 2024-11-26T15:25:08,664 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/B is initiating minor compaction (all files) 2024-11-26T15:25:08,664 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/B in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:08,664 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/7ce38b74b1174457baf875abd9ae6ad9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/9a0848a5a6be4862a23aeb689d263302, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b1b4943bab5a40debef771c593ca5bf1] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=35.5 K 2024-11-26T15:25:08,665 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44e26d299e4246689290db787269edb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732634707119 2024-11-26T15:25:08,665 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ce38b74b1174457baf875abd9ae6ad9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732634705741 2024-11-26T15:25:08,666 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2de7a7b5c0848109ffd5d0703b09815, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732634707795 2024-11-26T15:25:08,669 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a0848a5a6be4862a23aeb689d263302, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732634707119 2024-11-26T15:25:08,671 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting b1b4943bab5a40debef771c593ca5bf1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732634707795 2024-11-26T15:25:08,691 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#A#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:08,691 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/5a931432ba5d4aa082dc95c1b3a89df5 is 50, key is test_row_0/A:col10/1732634708471/Put/seqid=0 2024-11-26T15:25:08,705 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#B#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:08,706 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/a5c988e6889347eab143dc15cbae1e6f is 50, key is test_row_0/B:col10/1732634708471/Put/seqid=0 2024-11-26T15:25:08,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741864_1040 (size=12409) 2024-11-26T15:25:08,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741865_1041 (size=12409) 2024-11-26T15:25:08,774 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/a5c988e6889347eab143dc15cbae1e6f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/a5c988e6889347eab143dc15cbae1e6f 2024-11-26T15:25:08,794 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/B of 4c75b81af65e8ac430a5b8a152e75175 into a5c988e6889347eab143dc15cbae1e6f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:08,794 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:08,794 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/B, priority=13, startTime=1732634708658; duration=0sec 2024-11-26T15:25:08,795 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:08,795 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:B 2024-11-26T15:25:08,795 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:08,800 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:08,800 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/C is initiating minor compaction (all files) 2024-11-26T15:25:08,800 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/C in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:08,800 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/bd5d1ff10db247de9fc49bc7ec5bcaf8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/41345d97520b41f98e431298e6efb117, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/d7abe8e46905409c8489b83dd3247985] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=35.5 K 2024-11-26T15:25:08,801 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting bd5d1ff10db247de9fc49bc7ec5bcaf8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732634705741 2024-11-26T15:25:08,802 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 41345d97520b41f98e431298e6efb117, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732634707119 2024-11-26T15:25:08,803 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting d7abe8e46905409c8489b83dd3247985, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732634707795 2024-11-26T15:25:08,838 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#C#compaction#29 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:08,839 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/59d359a4b7444842927ca2fa9f3178ee is 50, key is test_row_0/C:col10/1732634708471/Put/seqid=0 2024-11-26T15:25:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:08,848 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:08,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-26T15:25:08,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/966cb6a3ccbf4b07b41272d975d39f9b is 50, key is test_row_0/A:col10/1732634708520/Put/seqid=0 2024-11-26T15:25:08,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741866_1042 (size=12409) 2024-11-26T15:25:08,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634768874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634768874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634768875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634768882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741867_1043 (size=12151) 2024-11-26T15:25:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634768883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:08,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/966cb6a3ccbf4b07b41272d975d39f9b 2024-11-26T15:25:08,903 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/59d359a4b7444842927ca2fa9f3178ee as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/59d359a4b7444842927ca2fa9f3178ee 2024-11-26T15:25:08,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/460f04fc22364a9ea9d3bc082403f4af is 50, key is test_row_0/B:col10/1732634708520/Put/seqid=0 2024-11-26T15:25:08,925 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/C of 4c75b81af65e8ac430a5b8a152e75175 into 59d359a4b7444842927ca2fa9f3178ee(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:08,925 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:08,925 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/C, priority=13, startTime=1732634708658; duration=0sec 2024-11-26T15:25:08,926 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:08,926 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:C 2024-11-26T15:25:08,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741868_1044 (size=12151) 2024-11-26T15:25:08,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/460f04fc22364a9ea9d3bc082403f4af 2024-11-26T15:25:08,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/ddce9c9e25d6471597f957fd1fe74f6a is 50, key is test_row_0/C:col10/1732634708520/Put/seqid=0 2024-11-26T15:25:08,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634768993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634768994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634768993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634768994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634768994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741869_1045 (size=12151) 2024-11-26T15:25:09,128 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/5a931432ba5d4aa082dc95c1b3a89df5 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5a931432ba5d4aa082dc95c1b3a89df5 2024-11-26T15:25:09,138 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/A of 4c75b81af65e8ac430a5b8a152e75175 into 5a931432ba5d4aa082dc95c1b3a89df5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:09,139 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:09,139 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/A, priority=13, startTime=1732634708657; duration=0sec 2024-11-26T15:25:09,139 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:09,139 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:A 2024-11-26T15:25:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T15:25:09,155 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:09,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:25:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-26T15:25:09,160 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:25:09,162 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:25:09,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:25:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T15:25:09,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634769213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634769212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634769213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634769213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634769214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T15:25:09,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T15:25:09,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:09,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:09,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:09,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:09,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:09,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/ddce9c9e25d6471597f957fd1fe74f6a 2024-11-26T15:25:09,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/966cb6a3ccbf4b07b41272d975d39f9b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/966cb6a3ccbf4b07b41272d975d39f9b 2024-11-26T15:25:09,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/966cb6a3ccbf4b07b41272d975d39f9b, entries=150, sequenceid=157, filesize=11.9 K 2024-11-26T15:25:09,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/460f04fc22364a9ea9d3bc082403f4af as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/460f04fc22364a9ea9d3bc082403f4af 2024-11-26T15:25:09,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/460f04fc22364a9ea9d3bc082403f4af, entries=150, sequenceid=157, filesize=11.9 K 2024-11-26T15:25:09,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/ddce9c9e25d6471597f957fd1fe74f6a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/ddce9c9e25d6471597f957fd1fe74f6a 2024-11-26T15:25:09,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T15:25:09,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:09,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:09,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:09,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:09,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T15:25:09,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/ddce9c9e25d6471597f957fd1fe74f6a, entries=150, sequenceid=157, filesize=11.9 K 2024-11-26T15:25:09,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 4c75b81af65e8ac430a5b8a152e75175 in 669ms, sequenceid=157, compaction requested=false 2024-11-26T15:25:09,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:09,528 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:09,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-26T15:25:09,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/cef9c09625e8458c8592b2d2dfe3af8c is 50, key is test_row_0/A:col10/1732634708880/Put/seqid=0 2024-11-26T15:25:09,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634769587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634769587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634769587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741870_1046 (size=14541) 2024-11-26T15:25:09,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634769595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634769595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T15:25:09,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:09,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:09,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:09,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:09,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:09,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634769703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634769703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634769704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634769713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634769713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T15:25:09,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:09,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:09,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:09,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:09,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T15:25:09,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634769913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634769913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634769913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634769923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634769923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:09,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T15:25:09,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:09,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:09,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:09,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:09,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:10,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/cef9c09625e8458c8592b2d2dfe3af8c 2024-11-26T15:25:10,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/b107a2e7ef414ae482b1c14c122fcce8 is 50, key is test_row_0/B:col10/1732634708880/Put/seqid=0 2024-11-26T15:25:10,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741871_1047 (size=12151) 2024-11-26T15:25:10,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T15:25:10,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:10,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:10,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:10,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:10,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:10,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:10,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634770223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:10,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634770223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:10,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634770233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:10,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:10,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634770242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:10,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:10,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634770243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:10,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:10,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T15:25:10,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:10,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:10,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:10,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:10,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T15:25:10,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T15:25:10,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:10,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:10,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:10,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:10,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:10,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/b107a2e7ef414ae482b1c14c122fcce8 2024-11-26T15:25:10,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/4b08bfd5ac9a45e09609f772849cbacc is 50, key is test_row_0/C:col10/1732634708880/Put/seqid=0 2024-11-26T15:25:10,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741872_1048 (size=12151) 2024-11-26T15:25:10,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/4b08bfd5ac9a45e09609f772849cbacc 2024-11-26T15:25:10,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/cef9c09625e8458c8592b2d2dfe3af8c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/cef9c09625e8458c8592b2d2dfe3af8c 2024-11-26T15:25:10,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/cef9c09625e8458c8592b2d2dfe3af8c, entries=200, sequenceid=174, filesize=14.2 K 2024-11-26T15:25:10,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/b107a2e7ef414ae482b1c14c122fcce8 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b107a2e7ef414ae482b1c14c122fcce8 2024-11-26T15:25:10,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b107a2e7ef414ae482b1c14c122fcce8, entries=150, sequenceid=174, filesize=11.9 K 2024-11-26T15:25:10,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/4b08bfd5ac9a45e09609f772849cbacc as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4b08bfd5ac9a45e09609f772849cbacc 2024-11-26T15:25:10,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T15:25:10,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:10,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:10,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:10,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:10,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4b08bfd5ac9a45e09609f772849cbacc, entries=150, sequenceid=174, filesize=11.9 K 2024-11-26T15:25:10,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 4c75b81af65e8ac430a5b8a152e75175 in 1045ms, sequenceid=174, compaction requested=true 2024-11-26T15:25:10,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:10,574 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:10,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:10,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:10,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:10,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:10,574 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:10,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:10,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:10,576 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:10,576 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/A is initiating minor compaction (all files) 2024-11-26T15:25:10,576 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:10,576 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/A in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:10,576 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/B is initiating minor compaction (all files) 2024-11-26T15:25:10,576 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/B in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:10,576 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5a931432ba5d4aa082dc95c1b3a89df5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/966cb6a3ccbf4b07b41272d975d39f9b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/cef9c09625e8458c8592b2d2dfe3af8c] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=38.2 K 2024-11-26T15:25:10,576 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/a5c988e6889347eab143dc15cbae1e6f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/460f04fc22364a9ea9d3bc082403f4af, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b107a2e7ef414ae482b1c14c122fcce8] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=35.9 K 2024-11-26T15:25:10,579 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a931432ba5d4aa082dc95c1b3a89df5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732634707795 2024-11-26T15:25:10,581 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 966cb6a3ccbf4b07b41272d975d39f9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732634708520 2024-11-26T15:25:10,581 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting a5c988e6889347eab143dc15cbae1e6f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732634707795 2024-11-26T15:25:10,582 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 460f04fc22364a9ea9d3bc082403f4af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732634708520 2024-11-26T15:25:10,582 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef9c09625e8458c8592b2d2dfe3af8c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732634708867 2024-11-26T15:25:10,582 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting b107a2e7ef414ae482b1c14c122fcce8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732634708867 2024-11-26T15:25:10,605 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:10,605 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#A#compaction#37 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:10,606 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/ecf37df8a6da4cdc81cec1abfa500d3b is 50, key is test_row_0/A:col10/1732634708880/Put/seqid=0 2024-11-26T15:25:10,607 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/8df8a0d6a4ac40d8ac85b2336ab863fe is 50, key is test_row_0/B:col10/1732634708880/Put/seqid=0 2024-11-26T15:25:10,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741874_1050 (size=12561) 2024-11-26T15:25:10,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741873_1049 (size=12561) 2024-11-26T15:25:10,644 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/8df8a0d6a4ac40d8ac85b2336ab863fe as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/8df8a0d6a4ac40d8ac85b2336ab863fe 2024-11-26T15:25:10,653 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/B of 4c75b81af65e8ac430a5b8a152e75175 into 8df8a0d6a4ac40d8ac85b2336ab863fe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:10,653 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:10,653 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/B, priority=13, startTime=1732634710574; duration=0sec 2024-11-26T15:25:10,653 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:10,654 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:B 2024-11-26T15:25:10,654 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:10,655 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:10,655 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/C is initiating minor compaction (all files) 2024-11-26T15:25:10,655 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/C in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:10,656 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/59d359a4b7444842927ca2fa9f3178ee, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/ddce9c9e25d6471597f957fd1fe74f6a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4b08bfd5ac9a45e09609f772849cbacc] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=35.9 K 2024-11-26T15:25:10,658 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 59d359a4b7444842927ca2fa9f3178ee, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732634707795 2024-11-26T15:25:10,660 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting ddce9c9e25d6471597f957fd1fe74f6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732634708520 2024-11-26T15:25:10,660 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b08bfd5ac9a45e09609f772849cbacc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732634708867 2024-11-26T15:25:10,689 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#C#compaction#38 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:10,690 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/2337cc20c3db4088bdbf167569197172 is 50, key is test_row_0/C:col10/1732634708880/Put/seqid=0 2024-11-26T15:25:10,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T15:25:10,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:10,722 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-26T15:25:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:10,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:10,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741875_1051 (size=12561) 2024-11-26T15:25:10,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/602edad5a2554615be3fe0e32266e287 is 50, key is test_row_0/A:col10/1732634709576/Put/seqid=0 2024-11-26T15:25:10,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:10,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634770767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:10,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:10,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634770767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634770769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:10,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634770771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:10,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634770770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:10,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:10,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741876_1052 (size=12151) 2024-11-26T15:25:10,801 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/602edad5a2554615be3fe0e32266e287 2024-11-26T15:25:10,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/53211272e7ea419186f4d53dd41c8ae6 is 50, key is test_row_0/B:col10/1732634709576/Put/seqid=0 2024-11-26T15:25:10,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741877_1053 (size=12151) 2024-11-26T15:25:10,871 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/53211272e7ea419186f4d53dd41c8ae6 2024-11-26T15:25:10,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634770883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:10,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634770883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/96e76b1bcf3d4b629a7456cbe95f6363 is 50, key is test_row_0/C:col10/1732634709576/Put/seqid=0 2024-11-26T15:25:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634770883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:10,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634770893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:10,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634770894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:10,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741878_1054 (size=12151) 2024-11-26T15:25:11,050 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/ecf37df8a6da4cdc81cec1abfa500d3b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/ecf37df8a6da4cdc81cec1abfa500d3b 2024-11-26T15:25:11,073 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/A of 4c75b81af65e8ac430a5b8a152e75175 into ecf37df8a6da4cdc81cec1abfa500d3b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:11,074 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:11,074 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/A, priority=13, startTime=1732634710574; duration=0sec 2024-11-26T15:25:11,074 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:11,074 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:A 2024-11-26T15:25:11,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634771092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634771092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634771094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634771104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634771103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,168 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/2337cc20c3db4088bdbf167569197172 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/2337cc20c3db4088bdbf167569197172 2024-11-26T15:25:11,179 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/C of 4c75b81af65e8ac430a5b8a152e75175 into 2337cc20c3db4088bdbf167569197172(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:11,179 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:11,179 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/C, priority=13, startTime=1732634710574; duration=0sec 2024-11-26T15:25:11,179 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:11,179 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:C 2024-11-26T15:25:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T15:25:11,336 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/96e76b1bcf3d4b629a7456cbe95f6363 2024-11-26T15:25:11,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/602edad5a2554615be3fe0e32266e287 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/602edad5a2554615be3fe0e32266e287 2024-11-26T15:25:11,363 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/602edad5a2554615be3fe0e32266e287, entries=150, sequenceid=197, filesize=11.9 K 2024-11-26T15:25:11,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/53211272e7ea419186f4d53dd41c8ae6 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/53211272e7ea419186f4d53dd41c8ae6 2024-11-26T15:25:11,374 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/53211272e7ea419186f4d53dd41c8ae6, entries=150, sequenceid=197, filesize=11.9 K 2024-11-26T15:25:11,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/96e76b1bcf3d4b629a7456cbe95f6363 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/96e76b1bcf3d4b629a7456cbe95f6363 2024-11-26T15:25:11,394 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/96e76b1bcf3d4b629a7456cbe95f6363, entries=150, sequenceid=197, filesize=11.9 K 2024-11-26T15:25:11,396 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 4c75b81af65e8ac430a5b8a152e75175 in 674ms, sequenceid=197, compaction requested=false 2024-11-26T15:25:11,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:11,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:11,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-26T15:25:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-26T15:25:11,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-26T15:25:11,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2370 sec 2024-11-26T15:25:11,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.2510 sec 2024-11-26T15:25:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:11,412 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:11,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-26T15:25:11,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/5a8e35f9a9854021aca9c9276df66447 is 50, key is test_row_0/A:col10/1732634710767/Put/seqid=0 2024-11-26T15:25:11,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741879_1055 (size=12151) 2024-11-26T15:25:11,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634771445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634771450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634771455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634771457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634771462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634771563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634771572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634771572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634771572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634771573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634771772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634771783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634771783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634771783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634771783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:11,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:11,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:11,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/5a8e35f9a9854021aca9c9276df66447 2024-11-26T15:25:11,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/0f350d9fa73a4612aded96e12511cb94 is 50, key is test_row_0/B:col10/1732634710767/Put/seqid=0 2024-11-26T15:25:11,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741880_1056 (size=12151) 2024-11-26T15:25:11,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/0f350d9fa73a4612aded96e12511cb94 2024-11-26T15:25:11,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/667ff3bcd977452889fe4a27431b99fc is 50, key is test_row_0/C:col10/1732634710767/Put/seqid=0 2024-11-26T15:25:11,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741881_1057 (size=12151) 2024-11-26T15:25:12,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634772087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634772094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634772094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634772094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634772094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/667ff3bcd977452889fe4a27431b99fc 2024-11-26T15:25:12,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/5a8e35f9a9854021aca9c9276df66447 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5a8e35f9a9854021aca9c9276df66447 2024-11-26T15:25:12,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5a8e35f9a9854021aca9c9276df66447, entries=150, sequenceid=217, filesize=11.9 K 2024-11-26T15:25:12,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/0f350d9fa73a4612aded96e12511cb94 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0f350d9fa73a4612aded96e12511cb94 2024-11-26T15:25:12,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0f350d9fa73a4612aded96e12511cb94, entries=150, sequenceid=217, filesize=11.9 K 2024-11-26T15:25:12,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/667ff3bcd977452889fe4a27431b99fc as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/667ff3bcd977452889fe4a27431b99fc 2024-11-26T15:25:12,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/667ff3bcd977452889fe4a27431b99fc, entries=150, sequenceid=217, filesize=11.9 K 2024-11-26T15:25:12,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 4c75b81af65e8ac430a5b8a152e75175 in 938ms, sequenceid=217, compaction requested=true 2024-11-26T15:25:12,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:12,351 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:12,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:12,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:12,351 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:12,353 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:12,353 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/B is initiating minor compaction (all files) 2024-11-26T15:25:12,353 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/B in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:12,353 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/8df8a0d6a4ac40d8ac85b2336ab863fe, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/53211272e7ea419186f4d53dd41c8ae6, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0f350d9fa73a4612aded96e12511cb94] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=36.0 K 2024-11-26T15:25:12,354 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:12,354 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/A is initiating minor compaction (all files) 2024-11-26T15:25:12,354 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/A in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:12,354 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 8df8a0d6a4ac40d8ac85b2336ab863fe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732634708867 2024-11-26T15:25:12,354 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/ecf37df8a6da4cdc81cec1abfa500d3b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/602edad5a2554615be3fe0e32266e287, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5a8e35f9a9854021aca9c9276df66447] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=36.0 K 2024-11-26T15:25:12,354 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 53211272e7ea419186f4d53dd41c8ae6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732634709576 2024-11-26T15:25:12,355 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecf37df8a6da4cdc81cec1abfa500d3b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732634708867 2024-11-26T15:25:12,356 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f350d9fa73a4612aded96e12511cb94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732634710767 2024-11-26T15:25:12,356 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 602edad5a2554615be3fe0e32266e287, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732634709576 2024-11-26T15:25:12,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:12,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:12,356 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a8e35f9a9854021aca9c9276df66447, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732634710767 2024-11-26T15:25:12,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:12,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:12,371 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#B#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:12,372 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/20595fe2e36443adbfd624e3b7c81ef0 is 50, key is test_row_0/B:col10/1732634710767/Put/seqid=0 2024-11-26T15:25:12,381 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#A#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:12,382 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/aea15a4b5f204179bf42486cf37938c9 is 50, key is test_row_0/A:col10/1732634710767/Put/seqid=0 2024-11-26T15:25:12,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741882_1058 (size=12663) 2024-11-26T15:25:12,428 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/20595fe2e36443adbfd624e3b7c81ef0 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/20595fe2e36443adbfd624e3b7c81ef0 2024-11-26T15:25:12,442 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/B of 4c75b81af65e8ac430a5b8a152e75175 into 20595fe2e36443adbfd624e3b7c81ef0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:12,442 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:12,442 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/B, priority=13, startTime=1732634712351; duration=0sec 2024-11-26T15:25:12,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741883_1059 (size=12663) 2024-11-26T15:25:12,443 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:12,443 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:B 2024-11-26T15:25:12,443 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:12,445 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:12,445 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/C is initiating minor compaction (all files) 2024-11-26T15:25:12,445 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/C in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:12,445 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/2337cc20c3db4088bdbf167569197172, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/96e76b1bcf3d4b629a7456cbe95f6363, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/667ff3bcd977452889fe4a27431b99fc] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=36.0 K 2024-11-26T15:25:12,446 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 2337cc20c3db4088bdbf167569197172, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732634708867 2024-11-26T15:25:12,457 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 96e76b1bcf3d4b629a7456cbe95f6363, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732634709576 2024-11-26T15:25:12,458 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 667ff3bcd977452889fe4a27431b99fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732634710767 2024-11-26T15:25:12,483 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#C#compaction#47 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:12,484 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/98e6c39193e84ed4a6d04692beef66fd is 50, key is test_row_0/C:col10/1732634710767/Put/seqid=0 2024-11-26T15:25:12,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741884_1060 (size=12663) 2024-11-26T15:25:12,524 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/98e6c39193e84ed4a6d04692beef66fd as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/98e6c39193e84ed4a6d04692beef66fd 2024-11-26T15:25:12,538 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/C of 4c75b81af65e8ac430a5b8a152e75175 into 98e6c39193e84ed4a6d04692beef66fd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:12,538 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:12,538 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/C, priority=13, startTime=1732634712356; duration=0sec 2024-11-26T15:25:12,539 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:12,539 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:C 2024-11-26T15:25:12,606 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:12,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-26T15:25:12,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/ae16636db9794f38a8ebabda2f3f2672 is 50, key is test_row_0/A:col10/1732634712605/Put/seqid=0 2024-11-26T15:25:12,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634772629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634772632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634772635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634772635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634772637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741885_1061 (size=16931) 2024-11-26T15:25:12,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/ae16636db9794f38a8ebabda2f3f2672 2024-11-26T15:25:12,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/618535f94d934e44bcf59fadf3e91f62 is 50, key is test_row_0/B:col10/1732634712605/Put/seqid=0 2024-11-26T15:25:12,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741886_1062 (size=12151) 2024-11-26T15:25:12,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634772743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634772743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634772743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634772752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634772753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,849 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/aea15a4b5f204179bf42486cf37938c9 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/aea15a4b5f204179bf42486cf37938c9 2024-11-26T15:25:12,857 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/A of 4c75b81af65e8ac430a5b8a152e75175 into aea15a4b5f204179bf42486cf37938c9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:12,857 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:12,857 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/A, priority=13, startTime=1732634712351; duration=0sec 2024-11-26T15:25:12,857 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:12,857 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:A 2024-11-26T15:25:12,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634772952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634772952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634772953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634772962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634772962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/618535f94d934e44bcf59fadf3e91f62 2024-11-26T15:25:13,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/9327b0e442334c0aae679b44521b3351 is 50, key is test_row_0/C:col10/1732634712605/Put/seqid=0 2024-11-26T15:25:13,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741887_1063 (size=12151) 2024-11-26T15:25:13,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/9327b0e442334c0aae679b44521b3351 2024-11-26T15:25:13,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/ae16636db9794f38a8ebabda2f3f2672 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/ae16636db9794f38a8ebabda2f3f2672 2024-11-26T15:25:13,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/ae16636db9794f38a8ebabda2f3f2672, entries=250, sequenceid=240, filesize=16.5 K 2024-11-26T15:25:13,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/618535f94d934e44bcf59fadf3e91f62 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/618535f94d934e44bcf59fadf3e91f62 2024-11-26T15:25:13,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/618535f94d934e44bcf59fadf3e91f62, entries=150, sequenceid=240, filesize=11.9 K 2024-11-26T15:25:13,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/9327b0e442334c0aae679b44521b3351 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/9327b0e442334c0aae679b44521b3351 2024-11-26T15:25:13,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/9327b0e442334c0aae679b44521b3351, entries=150, sequenceid=240, filesize=11.9 K 2024-11-26T15:25:13,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 4c75b81af65e8ac430a5b8a152e75175 in 551ms, sequenceid=240, compaction requested=false 2024-11-26T15:25:13,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:13,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:13,265 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:13,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-26T15:25:13,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/201a5ad9a4a54f238612e652bf2abd00 is 50, key is test_row_0/A:col10/1732634713264/Put/seqid=0 2024-11-26T15:25:13,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634773295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634773299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634773299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634773294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634773301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741888_1064 (size=14591) 2024-11-26T15:25:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T15:25:13,332 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:13,333 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:25:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-26T15:25:13,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T15:25:13,336 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:25:13,336 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:25:13,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:25:13,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634773412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634773412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634773413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634773413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634773413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T15:25:13,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T15:25:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:13,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:13,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634773623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634773623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634773623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634773623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634773623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T15:25:13,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:13,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:13,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:13,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:13,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T15:25:13,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/201a5ad9a4a54f238612e652bf2abd00 2024-11-26T15:25:13,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/ef865af10d5a4d1b9a58d2eccb8e5785 is 50, key is test_row_0/B:col10/1732634713264/Put/seqid=0 2024-11-26T15:25:13,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741889_1065 (size=12201) 2024-11-26T15:25:13,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/ef865af10d5a4d1b9a58d2eccb8e5785 2024-11-26T15:25:13,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T15:25:13,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:13,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:13,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:13,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:13,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:13,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:13,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/d307ed3a4fe24b928c60c7998b1f773b is 50, key is test_row_0/C:col10/1732634713264/Put/seqid=0 2024-11-26T15:25:13,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741890_1066 (size=12201) 2024-11-26T15:25:13,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634773932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634773933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634773933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634773933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:13,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634773933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:13,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T15:25:13,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:13,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:13,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:13,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:13,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T15:25:14,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T15:25:14,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:14,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:14,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:14,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:14,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:14,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/d307ed3a4fe24b928c60c7998b1f773b 2024-11-26T15:25:14,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/201a5ad9a4a54f238612e652bf2abd00 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/201a5ad9a4a54f238612e652bf2abd00 2024-11-26T15:25:14,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/201a5ad9a4a54f238612e652bf2abd00, entries=200, sequenceid=257, filesize=14.2 K 2024-11-26T15:25:14,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/ef865af10d5a4d1b9a58d2eccb8e5785 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/ef865af10d5a4d1b9a58d2eccb8e5785 2024-11-26T15:25:14,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/ef865af10d5a4d1b9a58d2eccb8e5785, entries=150, sequenceid=257, filesize=11.9 K 2024-11-26T15:25:14,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/d307ed3a4fe24b928c60c7998b1f773b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/d307ed3a4fe24b928c60c7998b1f773b 2024-11-26T15:25:14,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/d307ed3a4fe24b928c60c7998b1f773b, entries=150, sequenceid=257, filesize=11.9 K 2024-11-26T15:25:14,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 4c75b81af65e8ac430a5b8a152e75175 in 994ms, sequenceid=257, compaction requested=true 2024-11-26T15:25:14,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:14,260 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:14,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:14,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:14,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:14,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:14,260 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:14,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:14,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:14,261 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:14,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T15:25:14,261 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/A is initiating minor compaction (all files) 2024-11-26T15:25:14,261 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/A in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:14,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:14,261 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/aea15a4b5f204179bf42486cf37938c9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/ae16636db9794f38a8ebabda2f3f2672, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/201a5ad9a4a54f238612e652bf2abd00] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=43.1 K 2024-11-26T15:25:14,262 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-26T15:25:14,262 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:14,262 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/B is initiating minor compaction (all files) 2024-11-26T15:25:14,262 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/B in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:14,262 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/20595fe2e36443adbfd624e3b7c81ef0, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/618535f94d934e44bcf59fadf3e91f62, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/ef865af10d5a4d1b9a58d2eccb8e5785] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=36.1 K 2024-11-26T15:25:14,262 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting aea15a4b5f204179bf42486cf37938c9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732634710767 2024-11-26T15:25:14,262 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 20595fe2e36443adbfd624e3b7c81ef0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732634710767 2024-11-26T15:25:14,263 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae16636db9794f38a8ebabda2f3f2672, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732634711450 2024-11-26T15:25:14,263 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 618535f94d934e44bcf59fadf3e91f62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732634711452 2024-11-26T15:25:14,264 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting ef865af10d5a4d1b9a58d2eccb8e5785, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732634712632 2024-11-26T15:25:14,266 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 201a5ad9a4a54f238612e652bf2abd00, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732634712632 2024-11-26T15:25:14,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/07fcfa50eae7476f8a782424e6aed748 is 50, key is test_row_0/A:col10/1732634713297/Put/seqid=0 2024-11-26T15:25:14,279 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:14,280 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/9020a071ba2f464b9595a682b7d0430c is 50, key is test_row_0/B:col10/1732634713264/Put/seqid=0 2024-11-26T15:25:14,302 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#A#compaction#56 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:14,302 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/8a9c19de2f464d839198f3f6c9e928d5 is 50, key is test_row_0/A:col10/1732634713264/Put/seqid=0 2024-11-26T15:25:14,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741891_1067 (size=12301) 2024-11-26T15:25:14,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741893_1069 (size=12815) 2024-11-26T15:25:14,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741892_1068 (size=12815) 2024-11-26T15:25:14,350 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/8a9c19de2f464d839198f3f6c9e928d5 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/8a9c19de2f464d839198f3f6c9e928d5 2024-11-26T15:25:14,364 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/A of 4c75b81af65e8ac430a5b8a152e75175 into 8a9c19de2f464d839198f3f6c9e928d5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:14,364 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:14,364 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/A, priority=13, startTime=1732634714259; duration=0sec 2024-11-26T15:25:14,364 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:14,364 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:A 2024-11-26T15:25:14,365 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:14,366 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:14,366 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/C is initiating minor compaction (all files) 2024-11-26T15:25:14,366 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/C in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:14,366 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/98e6c39193e84ed4a6d04692beef66fd, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/9327b0e442334c0aae679b44521b3351, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/d307ed3a4fe24b928c60c7998b1f773b] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=36.1 K 2024-11-26T15:25:14,369 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98e6c39193e84ed4a6d04692beef66fd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732634710767 2024-11-26T15:25:14,369 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9327b0e442334c0aae679b44521b3351, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732634711452 2024-11-26T15:25:14,370 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting d307ed3a4fe24b928c60c7998b1f773b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732634712632 2024-11-26T15:25:14,398 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#C#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:14,404 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/06fb766588f9436d8965c2f68b696ff8 is 50, key is test_row_0/C:col10/1732634713264/Put/seqid=0 2024-11-26T15:25:14,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741894_1070 (size=12815) 2024-11-26T15:25:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:14,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:14,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634774464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T15:25:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:14,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634774470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:14,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634774470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:14,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:14,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:14,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634774472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:14,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:14,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634774475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:14,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634774583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:14,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:14,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634774583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:14,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634774584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:14,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:14,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634774583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634774583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:14,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:14,716 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/07fcfa50eae7476f8a782424e6aed748 2024-11-26T15:25:14,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/0941b74ce7ce456ea6b8fb8a7bfd69c0 is 50, key is test_row_0/B:col10/1732634713297/Put/seqid=0 2024-11-26T15:25:14,762 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/9020a071ba2f464b9595a682b7d0430c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/9020a071ba2f464b9595a682b7d0430c 2024-11-26T15:25:14,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741895_1071 (size=12301) 2024-11-26T15:25:14,764 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/0941b74ce7ce456ea6b8fb8a7bfd69c0 2024-11-26T15:25:14,781 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/B of 4c75b81af65e8ac430a5b8a152e75175 into 9020a071ba2f464b9595a682b7d0430c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:14,781 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:14,781 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/B, priority=13, startTime=1732634714260; duration=0sec 2024-11-26T15:25:14,781 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:14,781 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:B 2024-11-26T15:25:14,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/a3bb1e7065cd494bb0635298fa0ac363 is 50, key is test_row_0/C:col10/1732634713297/Put/seqid=0 2024-11-26T15:25:14,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634774802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:14,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634774803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:14,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634774803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:14,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634774803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634774803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:14,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741896_1072 (size=12301) 2024-11-26T15:25:14,825 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/06fb766588f9436d8965c2f68b696ff8 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/06fb766588f9436d8965c2f68b696ff8 2024-11-26T15:25:14,832 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/C of 4c75b81af65e8ac430a5b8a152e75175 into 06fb766588f9436d8965c2f68b696ff8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:14,832 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:14,833 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/C, priority=13, startTime=1732634714260; duration=0sec 2024-11-26T15:25:14,833 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:14,833 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:C 2024-11-26T15:25:15,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634775116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:15,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634775117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:15,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634775117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:15,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634775117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634775117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:15,209 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/a3bb1e7065cd494bb0635298fa0ac363 2024-11-26T15:25:15,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/07fcfa50eae7476f8a782424e6aed748 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/07fcfa50eae7476f8a782424e6aed748 2024-11-26T15:25:15,229 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/07fcfa50eae7476f8a782424e6aed748, entries=150, sequenceid=278, filesize=12.0 K 2024-11-26T15:25:15,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/0941b74ce7ce456ea6b8fb8a7bfd69c0 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0941b74ce7ce456ea6b8fb8a7bfd69c0 2024-11-26T15:25:15,273 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0941b74ce7ce456ea6b8fb8a7bfd69c0, entries=150, sequenceid=278, filesize=12.0 K 2024-11-26T15:25:15,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/a3bb1e7065cd494bb0635298fa0ac363 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/a3bb1e7065cd494bb0635298fa0ac363 2024-11-26T15:25:15,308 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/a3bb1e7065cd494bb0635298fa0ac363, entries=150, sequenceid=278, filesize=12.0 K 2024-11-26T15:25:15,310 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 4c75b81af65e8ac430a5b8a152e75175 in 1049ms, sequenceid=278, compaction requested=false 2024-11-26T15:25:15,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:15,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:15,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-26T15:25:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-26T15:25:15,315 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-26T15:25:15,315 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9750 sec 2024-11-26T15:25:15,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.9830 sec 2024-11-26T15:25:15,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T15:25:15,483 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:15,484 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:25:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-26T15:25:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T15:25:15,487 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:25:15,488 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:25:15,489 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:25:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T15:25:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:15,625 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:15,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-26T15:25:15,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/039857bc9fae4e23afdf1c46e654267a is 50, key is test_row_0/A:col10/1732634714469/Put/seqid=0 2024-11-26T15:25:15,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T15:25:15,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:15,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:15,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:15,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:15,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:15,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741897_1073 (size=12301) 2024-11-26T15:25:15,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/039857bc9fae4e23afdf1c46e654267a 2024-11-26T15:25:15,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634775664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:15,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:15,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634775665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634775668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:15,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634775673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:15,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634775674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:15,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/7334346488db413d82946198fbe3b33a is 50, key is test_row_0/B:col10/1732634714469/Put/seqid=0 2024-11-26T15:25:15,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741898_1074 (size=12301) 2024-11-26T15:25:15,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/7334346488db413d82946198fbe3b33a 2024-11-26T15:25:15,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/4d3535166eb04bf6a0d23ba0c023db8f is 50, key is test_row_0/C:col10/1732634714469/Put/seqid=0 2024-11-26T15:25:15,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741899_1075 (size=12301) 2024-11-26T15:25:15,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/4d3535166eb04bf6a0d23ba0c023db8f 2024-11-26T15:25:15,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:15,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/039857bc9fae4e23afdf1c46e654267a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/039857bc9fae4e23afdf1c46e654267a 2024-11-26T15:25:15,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634775773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:15,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:15,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/039857bc9fae4e23afdf1c46e654267a, entries=150, sequenceid=299, filesize=12.0 K 2024-11-26T15:25:15,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634775782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/7334346488db413d82946198fbe3b33a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/7334346488db413d82946198fbe3b33a 2024-11-26T15:25:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634775783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:15,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:15,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T15:25:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634775793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:15,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:15,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:15,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:15,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:15,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634775793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:15,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:15,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:15,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T15:25:15,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/7334346488db413d82946198fbe3b33a, entries=150, sequenceid=299, filesize=12.0 K 2024-11-26T15:25:15,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/4d3535166eb04bf6a0d23ba0c023db8f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4d3535166eb04bf6a0d23ba0c023db8f 2024-11-26T15:25:15,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4d3535166eb04bf6a0d23ba0c023db8f, entries=150, sequenceid=299, filesize=12.0 K 2024-11-26T15:25:15,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 4c75b81af65e8ac430a5b8a152e75175 in 194ms, sequenceid=299, compaction requested=true 2024-11-26T15:25:15,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:15,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:15,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:15,819 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:15,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:15,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:15,820 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:15,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:15,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:15,821 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:15,821 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/B is initiating minor compaction (all files) 2024-11-26T15:25:15,821 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/B in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:15,822 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/9020a071ba2f464b9595a682b7d0430c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0941b74ce7ce456ea6b8fb8a7bfd69c0, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/7334346488db413d82946198fbe3b33a] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=36.5 K 2024-11-26T15:25:15,822 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:15,822 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/A is initiating minor compaction (all files) 2024-11-26T15:25:15,822 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/A in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:15,822 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/8a9c19de2f464d839198f3f6c9e928d5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/07fcfa50eae7476f8a782424e6aed748, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/039857bc9fae4e23afdf1c46e654267a] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=36.5 K 2024-11-26T15:25:15,825 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a9c19de2f464d839198f3f6c9e928d5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732634712632 2024-11-26T15:25:15,825 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 9020a071ba2f464b9595a682b7d0430c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732634712632 2024-11-26T15:25:15,826 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07fcfa50eae7476f8a782424e6aed748, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732634713285 2024-11-26T15:25:15,826 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 0941b74ce7ce456ea6b8fb8a7bfd69c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732634713285 2024-11-26T15:25:15,826 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 039857bc9fae4e23afdf1c46e654267a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732634714469 2024-11-26T15:25:15,826 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 7334346488db413d82946198fbe3b33a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732634714469 2024-11-26T15:25:15,850 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#A#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:15,851 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/e7a8420b29734cf1be9cf9f1268a106c is 50, key is test_row_0/A:col10/1732634714469/Put/seqid=0 2024-11-26T15:25:15,855 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#B#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:15,856 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/1f9eaae662f443f595d4625d521d25bf is 50, key is test_row_0/B:col10/1732634714469/Put/seqid=0 2024-11-26T15:25:15,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741901_1077 (size=13017) 2024-11-26T15:25:15,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741900_1076 (size=13017) 2024-11-26T15:25:15,930 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/1f9eaae662f443f595d4625d521d25bf as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1f9eaae662f443f595d4625d521d25bf 2024-11-26T15:25:15,937 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/e7a8420b29734cf1be9cf9f1268a106c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/e7a8420b29734cf1be9cf9f1268a106c 2024-11-26T15:25:15,938 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/B of 4c75b81af65e8ac430a5b8a152e75175 into 1f9eaae662f443f595d4625d521d25bf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:15,938 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:15,938 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/B, priority=13, startTime=1732634715819; duration=0sec 2024-11-26T15:25:15,939 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:15,939 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:B 2024-11-26T15:25:15,939 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:15,942 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:15,942 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/C is initiating minor compaction (all files) 2024-11-26T15:25:15,942 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/C in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:15,942 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/06fb766588f9436d8965c2f68b696ff8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/a3bb1e7065cd494bb0635298fa0ac363, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4d3535166eb04bf6a0d23ba0c023db8f] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=36.5 K 2024-11-26T15:25:15,943 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 06fb766588f9436d8965c2f68b696ff8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732634712632 2024-11-26T15:25:15,944 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting a3bb1e7065cd494bb0635298fa0ac363, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732634713285 2024-11-26T15:25:15,944 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d3535166eb04bf6a0d23ba0c023db8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732634714469 2024-11-26T15:25:15,952 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/A of 4c75b81af65e8ac430a5b8a152e75175 into e7a8420b29734cf1be9cf9f1268a106c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:15,952 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:15,952 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/A, priority=13, startTime=1732634715819; duration=0sec 2024-11-26T15:25:15,952 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:15,952 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:A 2024-11-26T15:25:15,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T15:25:15,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:15,953 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-26T15:25:15,956 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#C#compaction#65 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:15,957 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/4797669add244d568f36ccd784914468 is 50, key is test_row_0/C:col10/1732634714469/Put/seqid=0 2024-11-26T15:25:15,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/c097310b5b30437d9cabd10a8c4e4a61 is 50, key is test_row_0/A:col10/1732634715663/Put/seqid=0 2024-11-26T15:25:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:15,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:16,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741902_1078 (size=13017) 2024-11-26T15:25:16,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741903_1079 (size=12301) 2024-11-26T15:25:16,009 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/c097310b5b30437d9cabd10a8c4e4a61 2024-11-26T15:25:16,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/1ef5c702c2264768a6154ebb5dd6db10 is 50, key is test_row_0/B:col10/1732634715663/Put/seqid=0 2024-11-26T15:25:16,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634776022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634776026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634776029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634776029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634776032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741904_1080 (size=12301) 2024-11-26T15:25:16,063 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/1ef5c702c2264768a6154ebb5dd6db10 2024-11-26T15:25:16,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/88677048548241b1b5642102a46429e1 is 50, key is test_row_0/C:col10/1732634715663/Put/seqid=0 2024-11-26T15:25:16,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741905_1081 (size=12301) 2024-11-26T15:25:16,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T15:25:16,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634776143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634776143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634776143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634776143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634776153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634776353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634776353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634776353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634776353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634776363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,412 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/4797669add244d568f36ccd784914468 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4797669add244d568f36ccd784914468 2024-11-26T15:25:16,421 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/C of 4c75b81af65e8ac430a5b8a152e75175 into 4797669add244d568f36ccd784914468(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:16,421 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:16,421 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/C, priority=13, startTime=1732634715820; duration=0sec 2024-11-26T15:25:16,421 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:16,421 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:C 2024-11-26T15:25:16,501 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/88677048548241b1b5642102a46429e1 2024-11-26T15:25:16,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/c097310b5b30437d9cabd10a8c4e4a61 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/c097310b5b30437d9cabd10a8c4e4a61 2024-11-26T15:25:16,514 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/c097310b5b30437d9cabd10a8c4e4a61, entries=150, sequenceid=320, filesize=12.0 K 2024-11-26T15:25:16,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/1ef5c702c2264768a6154ebb5dd6db10 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1ef5c702c2264768a6154ebb5dd6db10 2024-11-26T15:25:16,522 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1ef5c702c2264768a6154ebb5dd6db10, entries=150, sequenceid=320, filesize=12.0 K 2024-11-26T15:25:16,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/88677048548241b1b5642102a46429e1 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/88677048548241b1b5642102a46429e1 2024-11-26T15:25:16,529 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/88677048548241b1b5642102a46429e1, entries=150, sequenceid=320, filesize=12.0 K 2024-11-26T15:25:16,530 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 4c75b81af65e8ac430a5b8a152e75175 in 577ms, sequenceid=320, compaction requested=false 2024-11-26T15:25:16,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:16,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:16,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-26T15:25:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-26T15:25:16,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-26T15:25:16,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0430 sec 2024-11-26T15:25:16,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.0510 sec 2024-11-26T15:25:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T15:25:16,623 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:16,624 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:25:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-26T15:25:16,628 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:25:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T15:25:16,630 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:25:16,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:25:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:16,666 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:16,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-26T15:25:16,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/f7af43d7d08f4ffa81c879ef96b954fd is 50, key is test_row_0/A:col10/1732634716028/Put/seqid=0 2024-11-26T15:25:16,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634776688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634776691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634776691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634776704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634776704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741906_1082 (size=12301) 2024-11-26T15:25:16,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/f7af43d7d08f4ffa81c879ef96b954fd 2024-11-26T15:25:16,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T15:25:16,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/1f8d23b03e4a46d38172280ffa7c4b89 is 50, key is test_row_0/B:col10/1732634716028/Put/seqid=0 2024-11-26T15:25:16,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:16,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:16,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:16,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:16,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:16,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:16,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634776802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634776813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634776813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634776816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741907_1083 (size=12301) 2024-11-26T15:25:16,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634776822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T15:25:16,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:16,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:16,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:16,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:16,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:16,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634777012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634777023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634777023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634777024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634777032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:17,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:17,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:17,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:17,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/1f8d23b03e4a46d38172280ffa7c4b89 2024-11-26T15:25:17,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/5f6cd0ad11ac4c2bb62a0e4f21febb9c is 50, key is test_row_0/C:col10/1732634716028/Put/seqid=0 2024-11-26T15:25:17,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:17,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:17,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:17,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:17,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741908_1084 (size=12301) 2024-11-26T15:25:17,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/5f6cd0ad11ac4c2bb62a0e4f21febb9c 2024-11-26T15:25:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T15:25:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/f7af43d7d08f4ffa81c879ef96b954fd as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/f7af43d7d08f4ffa81c879ef96b954fd 2024-11-26T15:25:17,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/f7af43d7d08f4ffa81c879ef96b954fd, entries=150, sequenceid=340, filesize=12.0 K 2024-11-26T15:25:17,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/1f8d23b03e4a46d38172280ffa7c4b89 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1f8d23b03e4a46d38172280ffa7c4b89 2024-11-26T15:25:17,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1f8d23b03e4a46d38172280ffa7c4b89, entries=150, sequenceid=340, filesize=12.0 K 2024-11-26T15:25:17,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/5f6cd0ad11ac4c2bb62a0e4f21febb9c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/5f6cd0ad11ac4c2bb62a0e4f21febb9c 2024-11-26T15:25:17,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/5f6cd0ad11ac4c2bb62a0e4f21febb9c, entries=150, sequenceid=340, filesize=12.0 K 2024-11-26T15:25:17,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 4c75b81af65e8ac430a5b8a152e75175 in 632ms, sequenceid=340, compaction requested=true 2024-11-26T15:25:17,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:17,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:17,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:17,298 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:17,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:17,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:17,298 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:17,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:17,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:17,300 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:17,300 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:17,300 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/B is initiating minor compaction (all files) 2024-11-26T15:25:17,300 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/A is initiating minor compaction (all files) 2024-11-26T15:25:17,300 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/A in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:17,300 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/B in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:17,300 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1f9eaae662f443f595d4625d521d25bf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1ef5c702c2264768a6154ebb5dd6db10, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1f8d23b03e4a46d38172280ffa7c4b89] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=36.7 K 2024-11-26T15:25:17,300 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/e7a8420b29734cf1be9cf9f1268a106c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/c097310b5b30437d9cabd10a8c4e4a61, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/f7af43d7d08f4ffa81c879ef96b954fd] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=36.7 K 2024-11-26T15:25:17,300 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting e7a8420b29734cf1be9cf9f1268a106c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732634714469 2024-11-26T15:25:17,300 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f9eaae662f443f595d4625d521d25bf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732634714469 2024-11-26T15:25:17,301 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting c097310b5b30437d9cabd10a8c4e4a61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732634715663 2024-11-26T15:25:17,301 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ef5c702c2264768a6154ebb5dd6db10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732634715663 2024-11-26T15:25:17,301 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f8d23b03e4a46d38172280ffa7c4b89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732634716020 2024-11-26T15:25:17,301 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting f7af43d7d08f4ffa81c879ef96b954fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732634716020 2024-11-26T15:25:17,321 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:17,322 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#B#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:17,322 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/b22ae60f8cda4281aab5d1b75af5d8ee is 50, key is test_row_0/A:col10/1732634716028/Put/seqid=0 2024-11-26T15:25:17,322 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/0f8ea24d2a3a4a78863577475d045c39 is 50, key is test_row_0/B:col10/1732634716028/Put/seqid=0 2024-11-26T15:25:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:17,336 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:17,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-26T15:25:17,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741910_1086 (size=13119) 2024-11-26T15:25:17,352 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/0f8ea24d2a3a4a78863577475d045c39 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0f8ea24d2a3a4a78863577475d045c39 2024-11-26T15:25:17,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634777351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634777352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634777355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,362 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/B of 4c75b81af65e8ac430a5b8a152e75175 into 0f8ea24d2a3a4a78863577475d045c39(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:17,362 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:17,362 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/B, priority=13, startTime=1732634717298; duration=0sec 2024-11-26T15:25:17,362 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:17,362 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:B 2024-11-26T15:25:17,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,362 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634777355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634777357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,365 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:17,365 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/C is initiating minor compaction (all files) 2024-11-26T15:25:17,365 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/C in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:17,365 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4797669add244d568f36ccd784914468, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/88677048548241b1b5642102a46429e1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/5f6cd0ad11ac4c2bb62a0e4f21febb9c] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=36.7 K 2024-11-26T15:25:17,366 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4797669add244d568f36ccd784914468, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732634714469 2024-11-26T15:25:17,370 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88677048548241b1b5642102a46429e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732634715663 2024-11-26T15:25:17,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741909_1085 (size=13119) 2024-11-26T15:25:17,370 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f6cd0ad11ac4c2bb62a0e4f21febb9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732634716020 2024-11-26T15:25:17,379 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/b22ae60f8cda4281aab5d1b75af5d8ee as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b22ae60f8cda4281aab5d1b75af5d8ee 2024-11-26T15:25:17,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/a5e1f38271c049a3806004da88db5d1b is 50, key is test_row_0/A:col10/1732634717335/Put/seqid=0 2024-11-26T15:25:17,392 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/A of 4c75b81af65e8ac430a5b8a152e75175 into b22ae60f8cda4281aab5d1b75af5d8ee(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:17,392 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:17,392 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/A, priority=13, startTime=1732634717298; duration=0sec 2024-11-26T15:25:17,392 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:17,392 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:A 2024-11-26T15:25:17,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741911_1087 (size=14741) 2024-11-26T15:25:17,404 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#C#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:17,405 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/3d66adb7c70345cf82a97e79d98476ab is 50, key is test_row_0/C:col10/1732634716028/Put/seqid=0 2024-11-26T15:25:17,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/a5e1f38271c049a3806004da88db5d1b 2024-11-26T15:25:17,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:17,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:17,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:17,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:17,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/6533acc68b374f8d9b881d7bec6b5b7c is 50, key is test_row_0/B:col10/1732634717335/Put/seqid=0 2024-11-26T15:25:17,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741912_1088 (size=13119) 2024-11-26T15:25:17,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741913_1089 (size=12301) 2024-11-26T15:25:17,438 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/3d66adb7c70345cf82a97e79d98476ab as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/3d66adb7c70345cf82a97e79d98476ab 2024-11-26T15:25:17,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/6533acc68b374f8d9b881d7bec6b5b7c 2024-11-26T15:25:17,446 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/C of 4c75b81af65e8ac430a5b8a152e75175 into 3d66adb7c70345cf82a97e79d98476ab(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:17,446 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:17,446 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/C, priority=13, startTime=1732634717298; duration=0sec 2024-11-26T15:25:17,446 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:17,446 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:C 2024-11-26T15:25:17,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/90cf0566a7164d4cbe6ddb9cb245c454 is 50, key is test_row_0/C:col10/1732634717335/Put/seqid=0 2024-11-26T15:25:17,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741914_1090 (size=12301) 2024-11-26T15:25:17,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634777462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634777462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634777472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634777472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634777473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:17,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:17,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:17,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:17,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634777672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634777672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634777683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634777683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634777683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:17,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:17,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:17,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:17,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:17,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T15:25:17,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/90cf0566a7164d4cbe6ddb9cb245c454 2024-11-26T15:25:17,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/a5e1f38271c049a3806004da88db5d1b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/a5e1f38271c049a3806004da88db5d1b 2024-11-26T15:25:17,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:17,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:17,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/a5e1f38271c049a3806004da88db5d1b, entries=200, sequenceid=363, filesize=14.4 K 2024-11-26T15:25:17,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/6533acc68b374f8d9b881d7bec6b5b7c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/6533acc68b374f8d9b881d7bec6b5b7c 2024-11-26T15:25:17,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/6533acc68b374f8d9b881d7bec6b5b7c, entries=150, sequenceid=363, filesize=12.0 K 2024-11-26T15:25:17,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/90cf0566a7164d4cbe6ddb9cb245c454 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/90cf0566a7164d4cbe6ddb9cb245c454 2024-11-26T15:25:17,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/90cf0566a7164d4cbe6ddb9cb245c454, entries=150, sequenceid=363, filesize=12.0 K 2024-11-26T15:25:17,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 4c75b81af65e8ac430a5b8a152e75175 in 552ms, sequenceid=363, compaction requested=false 2024-11-26T15:25:17,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:17,984 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:17,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-26T15:25:17,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/dcf99a29c8234a5caae16582cbbaa322 is 50, key is test_row_0/A:col10/1732634717983/Put/seqid=0 2024-11-26T15:25:18,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741915_1091 (size=17181) 2024-11-26T15:25:18,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634778012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634778014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634778015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:18,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:18,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:18,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:18,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634778017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634778017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634778123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634778123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634778132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634778132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634778132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:18,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:18,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:18,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:18,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:18,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:18,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:18,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:18,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634778333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634778333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634778342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634778342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634778342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/dcf99a29c8234a5caae16582cbbaa322 2024-11-26T15:25:18,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/e30bd4f5b29f4945834886de56254952 is 50, key is test_row_0/B:col10/1732634717983/Put/seqid=0 2024-11-26T15:25:18,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741916_1092 (size=12301) 2024-11-26T15:25:18,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:18,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:18,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:18,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:18,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:18,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:18,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:18,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:18,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634778642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634778642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634778652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634778652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634778652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:18,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:18,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T15:25:18,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:18,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:18,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:18,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:18,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/e30bd4f5b29f4945834886de56254952 2024-11-26T15:25:18,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/c81d425850904caf855d1d4915995534 is 50, key is test_row_0/C:col10/1732634717983/Put/seqid=0 2024-11-26T15:25:18,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741917_1093 (size=12301) 2024-11-26T15:25:18,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:18,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:18,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:18,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:18,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:19,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:19,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:19,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:19,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:19,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:19,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:19,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634779152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:19,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634779152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:19,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634779163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:19,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634779163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:19,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634779172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:19,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:19,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:19,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:19,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/c81d425850904caf855d1d4915995534 2024-11-26T15:25:19,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:19,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:19,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:19,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:19,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:19,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:19,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:19,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/dcf99a29c8234a5caae16582cbbaa322 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/dcf99a29c8234a5caae16582cbbaa322 2024-11-26T15:25:19,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/dcf99a29c8234a5caae16582cbbaa322, entries=250, sequenceid=381, filesize=16.8 K 2024-11-26T15:25:19,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/e30bd4f5b29f4945834886de56254952 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/e30bd4f5b29f4945834886de56254952 2024-11-26T15:25:19,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/e30bd4f5b29f4945834886de56254952, entries=150, sequenceid=381, filesize=12.0 K 2024-11-26T15:25:19,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/c81d425850904caf855d1d4915995534 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/c81d425850904caf855d1d4915995534 2024-11-26T15:25:19,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/c81d425850904caf855d1d4915995534, entries=150, sequenceid=381, filesize=12.0 K 2024-11-26T15:25:19,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 4c75b81af65e8ac430a5b8a152e75175 in 1289ms, sequenceid=381, compaction requested=true 2024-11-26T15:25:19,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:19,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:19,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:19,273 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:19,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:19,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:19,273 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:19,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:19,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:19,274 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45041 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:19,274 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:19,274 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/B is initiating minor compaction (all files) 2024-11-26T15:25:19,274 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/A is initiating minor compaction (all files) 2024-11-26T15:25:19,275 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/A in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:19,275 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/B in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:19,275 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0f8ea24d2a3a4a78863577475d045c39, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/6533acc68b374f8d9b881d7bec6b5b7c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/e30bd4f5b29f4945834886de56254952] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=36.8 K 2024-11-26T15:25:19,275 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b22ae60f8cda4281aab5d1b75af5d8ee, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/a5e1f38271c049a3806004da88db5d1b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/dcf99a29c8234a5caae16582cbbaa322] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=44.0 K 2024-11-26T15:25:19,275 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f8ea24d2a3a4a78863577475d045c39, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732634716020 2024-11-26T15:25:19,275 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting b22ae60f8cda4281aab5d1b75af5d8ee, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732634716020 2024-11-26T15:25:19,276 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 6533acc68b374f8d9b881d7bec6b5b7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732634717328 2024-11-26T15:25:19,276 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5e1f38271c049a3806004da88db5d1b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732634716683 2024-11-26T15:25:19,276 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting e30bd4f5b29f4945834886de56254952, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732634717350 2024-11-26T15:25:19,276 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcf99a29c8234a5caae16582cbbaa322, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732634717350 2024-11-26T15:25:19,285 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#B#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:19,285 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/ce3af76a359541b09568d930a5b0f239 is 50, key is test_row_0/B:col10/1732634717983/Put/seqid=0 2024-11-26T15:25:19,286 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#A#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:19,287 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/e10a27fe23204277a22ba71a69e56029 is 50, key is test_row_0/A:col10/1732634717983/Put/seqid=0 2024-11-26T15:25:19,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741918_1094 (size=13221) 2024-11-26T15:25:19,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741919_1095 (size=13221) 2024-11-26T15:25:19,303 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/ce3af76a359541b09568d930a5b0f239 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/ce3af76a359541b09568d930a5b0f239 2024-11-26T15:25:19,308 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/e10a27fe23204277a22ba71a69e56029 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/e10a27fe23204277a22ba71a69e56029 2024-11-26T15:25:19,310 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/B of 4c75b81af65e8ac430a5b8a152e75175 into ce3af76a359541b09568d930a5b0f239(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:19,310 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:19,310 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/B, priority=13, startTime=1732634719273; duration=0sec 2024-11-26T15:25:19,311 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:19,311 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:B 2024-11-26T15:25:19,311 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:19,312 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:19,313 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/C is initiating minor compaction (all files) 2024-11-26T15:25:19,313 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/C in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:19,313 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/3d66adb7c70345cf82a97e79d98476ab, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/90cf0566a7164d4cbe6ddb9cb245c454, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/c81d425850904caf855d1d4915995534] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=36.8 K 2024-11-26T15:25:19,313 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d66adb7c70345cf82a97e79d98476ab, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732634716020 2024-11-26T15:25:19,314 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 90cf0566a7164d4cbe6ddb9cb245c454, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732634717328 2024-11-26T15:25:19,314 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting c81d425850904caf855d1d4915995534, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732634717350 2024-11-26T15:25:19,319 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/A of 4c75b81af65e8ac430a5b8a152e75175 into e10a27fe23204277a22ba71a69e56029(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:19,319 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:19,319 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/A, priority=13, startTime=1732634719273; duration=0sec 2024-11-26T15:25:19,319 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:19,319 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:A 2024-11-26T15:25:19,324 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#C#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:19,325 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/265cef5c0fe040c3a302704767a22f1e is 50, key is test_row_0/C:col10/1732634717983/Put/seqid=0 2024-11-26T15:25:19,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741920_1096 (size=13221) 2024-11-26T15:25:19,340 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/265cef5c0fe040c3a302704767a22f1e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/265cef5c0fe040c3a302704767a22f1e 2024-11-26T15:25:19,348 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/C of 4c75b81af65e8ac430a5b8a152e75175 into 265cef5c0fe040c3a302704767a22f1e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:19,349 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:19,349 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/C, priority=13, startTime=1732634719273; duration=0sec 2024-11-26T15:25:19,349 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:19,349 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:C 2024-11-26T15:25:19,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T15:25:19,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:19,404 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-26T15:25:19,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/42d427cc1b1f4625bec51f46981f92a9 is 50, key is test_row_0/A:col10/1732634718015/Put/seqid=0 2024-11-26T15:25:19,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741921_1097 (size=12301) 2024-11-26T15:25:19,415 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/42d427cc1b1f4625bec51f46981f92a9 2024-11-26T15:25:19,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/5dfe31b652ce4ff395a0e846bdc68adb is 50, key is test_row_0/B:col10/1732634718015/Put/seqid=0 2024-11-26T15:25:19,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741922_1098 (size=12301) 2024-11-26T15:25:19,836 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/5dfe31b652ce4ff395a0e846bdc68adb 2024-11-26T15:25:19,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/e0fe81cd6c8540ab9afd88fc9063ce29 is 50, key is test_row_0/C:col10/1732634718015/Put/seqid=0 2024-11-26T15:25:19,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741923_1099 (size=12301) 2024-11-26T15:25:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:20,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:20,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634780183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634780184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634780185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,191 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in 4c75b81af65e8ac430a5b8a152e75175 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:25:20,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634780189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634780191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,250 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/e0fe81cd6c8540ab9afd88fc9063ce29 2024-11-26T15:25:20,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/42d427cc1b1f4625bec51f46981f92a9 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/42d427cc1b1f4625bec51f46981f92a9 2024-11-26T15:25:20,263 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/42d427cc1b1f4625bec51f46981f92a9, entries=150, sequenceid=405, filesize=12.0 K 2024-11-26T15:25:20,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/5dfe31b652ce4ff395a0e846bdc68adb as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/5dfe31b652ce4ff395a0e846bdc68adb 2024-11-26T15:25:20,270 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/5dfe31b652ce4ff395a0e846bdc68adb, entries=150, sequenceid=405, filesize=12.0 K 2024-11-26T15:25:20,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/e0fe81cd6c8540ab9afd88fc9063ce29 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/e0fe81cd6c8540ab9afd88fc9063ce29 2024-11-26T15:25:20,276 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/e0fe81cd6c8540ab9afd88fc9063ce29, entries=150, sequenceid=405, filesize=12.0 K 2024-11-26T15:25:20,278 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 4c75b81af65e8ac430a5b8a152e75175 in 874ms, sequenceid=405, compaction requested=false 2024-11-26T15:25:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-26T15:25:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-26T15:25:20,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-26T15:25:20,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6490 sec 2024-11-26T15:25:20,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 3.6580 sec 2024-11-26T15:25:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:20,294 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:20,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-26T15:25:20,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/5c777c35e3324de18bf48d0a88a2344e is 50, key is test_row_0/A:col10/1732634720293/Put/seqid=0 2024-11-26T15:25:20,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741924_1100 (size=17181) 2024-11-26T15:25:20,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634780344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634780344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634780349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634780351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634780452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634780453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634780463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634780463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634780663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634780663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634780672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634780672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/5c777c35e3324de18bf48d0a88a2344e 2024-11-26T15:25:20,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/32d555a7efaa4f1a8eeccde024ab1965 is 50, key is test_row_0/B:col10/1732634720293/Put/seqid=0 2024-11-26T15:25:20,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741925_1101 (size=12301) 2024-11-26T15:25:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T15:25:20,793 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:20,793 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:25:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-26T15:25:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T15:25:20,833 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:25:20,834 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:25:20,834 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:25:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T15:25:20,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634780972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634780972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634780983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:20,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634780983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:20,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:20,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:20,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T15:25:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:20,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:20,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:20,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:20,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T15:25:21,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/32d555a7efaa4f1a8eeccde024ab1965 2024-11-26T15:25:21,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/a53078025da645a3b9fddf4afde228f1 is 50, key is test_row_0/C:col10/1732634720293/Put/seqid=0 2024-11-26T15:25:21,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T15:25:21,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:21,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:21,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:21,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:21,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:21,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:21,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741926_1102 (size=12301) 2024-11-26T15:25:21,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/a53078025da645a3b9fddf4afde228f1 2024-11-26T15:25:21,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/5c777c35e3324de18bf48d0a88a2344e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5c777c35e3324de18bf48d0a88a2344e 2024-11-26T15:25:21,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5c777c35e3324de18bf48d0a88a2344e, entries=250, sequenceid=422, filesize=16.8 K 2024-11-26T15:25:21,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/32d555a7efaa4f1a8eeccde024ab1965 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/32d555a7efaa4f1a8eeccde024ab1965 2024-11-26T15:25:21,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/32d555a7efaa4f1a8eeccde024ab1965, entries=150, sequenceid=422, filesize=12.0 K 2024-11-26T15:25:21,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/a53078025da645a3b9fddf4afde228f1 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/a53078025da645a3b9fddf4afde228f1 2024-11-26T15:25:21,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/a53078025da645a3b9fddf4afde228f1, entries=150, sequenceid=422, filesize=12.0 K 2024-11-26T15:25:21,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 4c75b81af65e8ac430a5b8a152e75175 in 903ms, sequenceid=422, compaction requested=true 2024-11-26T15:25:21,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:21,198 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:21,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:21,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:21,198 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:21,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:21,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:21,199 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:21,200 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/A is initiating minor compaction (all files) 2024-11-26T15:25:21,200 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/A in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:21,200 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/e10a27fe23204277a22ba71a69e56029, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/42d427cc1b1f4625bec51f46981f92a9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5c777c35e3324de18bf48d0a88a2344e] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=41.7 K 2024-11-26T15:25:21,200 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:21,201 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/B is initiating minor compaction (all files) 2024-11-26T15:25:21,201 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/B in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:21,201 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/ce3af76a359541b09568d930a5b0f239, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/5dfe31b652ce4ff395a0e846bdc68adb, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/32d555a7efaa4f1a8eeccde024ab1965] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=36.9 K 2024-11-26T15:25:21,202 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting ce3af76a359541b09568d930a5b0f239, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732634717350 2024-11-26T15:25:21,202 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting e10a27fe23204277a22ba71a69e56029, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732634717350 2024-11-26T15:25:21,203 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dfe31b652ce4ff395a0e846bdc68adb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732634718006 2024-11-26T15:25:21,203 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42d427cc1b1f4625bec51f46981f92a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732634718006 2024-11-26T15:25:21,203 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 32d555a7efaa4f1a8eeccde024ab1965, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732634720183 2024-11-26T15:25:21,203 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c777c35e3324de18bf48d0a88a2344e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732634720173 2024-11-26T15:25:21,226 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#B#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:21,227 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/a35e5d66c7b74e5199b521818b26aa5c is 50, key is test_row_0/B:col10/1732634720293/Put/seqid=0 2024-11-26T15:25:21,230 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#A#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:21,231 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/74ef4197c680458cae87edaf04716cb5 is 50, key is test_row_0/A:col10/1732634720293/Put/seqid=0 2024-11-26T15:25:21,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741928_1104 (size=13323) 2024-11-26T15:25:21,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741927_1103 (size=13323) 2024-11-26T15:25:21,265 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/a35e5d66c7b74e5199b521818b26aa5c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/a35e5d66c7b74e5199b521818b26aa5c 2024-11-26T15:25:21,277 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/B of 4c75b81af65e8ac430a5b8a152e75175 into a35e5d66c7b74e5199b521818b26aa5c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:21,277 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:21,277 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/B, priority=13, startTime=1732634721198; duration=0sec 2024-11-26T15:25:21,277 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:21,278 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:B 2024-11-26T15:25:21,278 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:21,288 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:21,288 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/C is initiating minor compaction (all files) 2024-11-26T15:25:21,288 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/C in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:21,288 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/265cef5c0fe040c3a302704767a22f1e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/e0fe81cd6c8540ab9afd88fc9063ce29, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/a53078025da645a3b9fddf4afde228f1] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=36.9 K 2024-11-26T15:25:21,289 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 265cef5c0fe040c3a302704767a22f1e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732634717350 2024-11-26T15:25:21,289 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting e0fe81cd6c8540ab9afd88fc9063ce29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732634718006 2024-11-26T15:25:21,289 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting a53078025da645a3b9fddf4afde228f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732634720183 2024-11-26T15:25:21,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T15:25:21,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:21,303 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-26T15:25:21,305 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#C#compaction#92 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:21,306 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/72befbf8b7c948b89d4bfe703d9b4823 is 50, key is test_row_0/C:col10/1732634720293/Put/seqid=0 2024-11-26T15:25:21,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/45fb0839161942cc9c22266f336e39aa is 50, key is test_row_0/A:col10/1732634720340/Put/seqid=0 2024-11-26T15:25:21,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741929_1105 (size=13323) 2024-11-26T15:25:21,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741930_1106 (size=12301) 2024-11-26T15:25:21,352 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/45fb0839161942cc9c22266f336e39aa 2024-11-26T15:25:21,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/142440c4927547d5a0da321a89c08f5f is 50, key is test_row_0/B:col10/1732634720340/Put/seqid=0 2024-11-26T15:25:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T15:25:21,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741931_1107 (size=12301) 2024-11-26T15:25:21,431 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/142440c4927547d5a0da321a89c08f5f 2024-11-26T15:25:21,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/b816948a14fa47cf828fde93e8ea90f1 is 50, key is test_row_0/C:col10/1732634720340/Put/seqid=0 2024-11-26T15:25:21,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741932_1108 (size=12301) 2024-11-26T15:25:21,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. as already flushing 2024-11-26T15:25:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:21,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634781513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:21,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:21,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634781515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:21,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634781518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:21,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634781518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:21,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:21,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:21,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634781623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:21,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634781633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:21,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634781633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:21,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634781634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:21,650 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/74ef4197c680458cae87edaf04716cb5 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/74ef4197c680458cae87edaf04716cb5 2024-11-26T15:25:21,659 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/A of 4c75b81af65e8ac430a5b8a152e75175 into 74ef4197c680458cae87edaf04716cb5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:21,659 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:21,659 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/A, priority=13, startTime=1732634721198; duration=0sec 2024-11-26T15:25:21,660 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:21,660 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:A 2024-11-26T15:25:21,739 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/72befbf8b7c948b89d4bfe703d9b4823 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/72befbf8b7c948b89d4bfe703d9b4823 2024-11-26T15:25:21,752 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/C of 4c75b81af65e8ac430a5b8a152e75175 into 72befbf8b7c948b89d4bfe703d9b4823(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:21,753 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:21,753 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/C, priority=13, startTime=1732634721199; duration=0sec 2024-11-26T15:25:21,754 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:21,754 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:C 2024-11-26T15:25:21,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634781833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:21,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634781842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:21,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634781842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:21,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:21,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634781843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:21,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:21,877 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/b816948a14fa47cf828fde93e8ea90f1 2024-11-26T15:25:21,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/45fb0839161942cc9c22266f336e39aa as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/45fb0839161942cc9c22266f336e39aa 2024-11-26T15:25:21,892 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/45fb0839161942cc9c22266f336e39aa, entries=150, sequenceid=442, filesize=12.0 K 2024-11-26T15:25:21,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/142440c4927547d5a0da321a89c08f5f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/142440c4927547d5a0da321a89c08f5f 2024-11-26T15:25:21,903 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/142440c4927547d5a0da321a89c08f5f, entries=150, sequenceid=442, filesize=12.0 K 2024-11-26T15:25:21,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/b816948a14fa47cf828fde93e8ea90f1 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/b816948a14fa47cf828fde93e8ea90f1 2024-11-26T15:25:21,925 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/b816948a14fa47cf828fde93e8ea90f1, entries=150, sequenceid=442, filesize=12.0 K 2024-11-26T15:25:21,927 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 4c75b81af65e8ac430a5b8a152e75175 in 624ms, sequenceid=442, compaction requested=false 2024-11-26T15:25:21,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:21,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:21,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-26T15:25:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-26T15:25:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T15:25:21,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-26T15:25:21,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0960 sec 2024-11-26T15:25:21,937 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.1410 sec 2024-11-26T15:25:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:22,144 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:22,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-26T15:25:22,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/d09b93f70d3f4b5e947fb8bc9e75e2bb is 50, key is test_row_0/A:col10/1732634721514/Put/seqid=0 2024-11-26T15:25:22,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:22,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634782169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634782169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:22,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741933_1109 (size=14741) 2024-11-26T15:25:22,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:22,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:22,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634782169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634782170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:22,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/d09b93f70d3f4b5e947fb8bc9e75e2bb 2024-11-26T15:25:22,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/79dee34798474fcabb07ae1e55b7dd56 is 50, key is test_row_0/B:col10/1732634721514/Put/seqid=0 2024-11-26T15:25:22,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1732634782203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:22,204 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in 4c75b81af65e8ac430a5b8a152e75175 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:25:22,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:22,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741934_1110 (size=12301) 2024-11-26T15:25:22,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634782283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:22,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:22,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634782283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:22,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:22,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634782283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:22,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634782284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:22,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47306 deadline: 1732634782493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:22,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:22,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47330 deadline: 1732634782493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:22,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:22,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:22,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47278 deadline: 1732634782493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:22,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:22,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47318 deadline: 1732634782493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:25:22,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/79dee34798474fcabb07ae1e55b7dd56 2024-11-26T15:25:22,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/bc5e5bbef35a44e599765dde5859bf4e is 50, key is test_row_0/C:col10/1732634721514/Put/seqid=0 2024-11-26T15:25:22,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741935_1111 (size=12301) 2024-11-26T15:25:22,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/bc5e5bbef35a44e599765dde5859bf4e 2024-11-26T15:25:22,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/d09b93f70d3f4b5e947fb8bc9e75e2bb as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/d09b93f70d3f4b5e947fb8bc9e75e2bb 2024-11-26T15:25:22,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/d09b93f70d3f4b5e947fb8bc9e75e2bb, entries=200, sequenceid=463, filesize=14.4 K 2024-11-26T15:25:22,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/79dee34798474fcabb07ae1e55b7dd56 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/79dee34798474fcabb07ae1e55b7dd56 2024-11-26T15:25:22,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/79dee34798474fcabb07ae1e55b7dd56, entries=150, sequenceid=463, filesize=12.0 K 2024-11-26T15:25:22,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/bc5e5bbef35a44e599765dde5859bf4e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/bc5e5bbef35a44e599765dde5859bf4e 2024-11-26T15:25:22,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/bc5e5bbef35a44e599765dde5859bf4e, entries=150, sequenceid=463, filesize=12.0 K 2024-11-26T15:25:22,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 4c75b81af65e8ac430a5b8a152e75175 in 553ms, sequenceid=463, compaction requested=true 2024-11-26T15:25:22,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:22,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:22,697 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:22,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:22,698 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:22,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:22,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:22,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c75b81af65e8ac430a5b8a152e75175:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:22,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:22,699 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:22,699 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/A is initiating minor compaction (all files) 2024-11-26T15:25:22,699 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/A in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:22,699 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/74ef4197c680458cae87edaf04716cb5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/45fb0839161942cc9c22266f336e39aa, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/d09b93f70d3f4b5e947fb8bc9e75e2bb] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=39.4 K 2024-11-26T15:25:22,700 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74ef4197c680458cae87edaf04716cb5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732634720183 2024-11-26T15:25:22,701 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:22,701 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/B is initiating minor compaction (all files) 2024-11-26T15:25:22,701 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/B in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:22,701 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/a35e5d66c7b74e5199b521818b26aa5c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/142440c4927547d5a0da321a89c08f5f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/79dee34798474fcabb07ae1e55b7dd56] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=37.0 K 2024-11-26T15:25:22,701 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting a35e5d66c7b74e5199b521818b26aa5c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732634720183 2024-11-26T15:25:22,702 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45fb0839161942cc9c22266f336e39aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1732634720340 2024-11-26T15:25:22,702 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 142440c4927547d5a0da321a89c08f5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1732634720340 2024-11-26T15:25:22,702 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 79dee34798474fcabb07ae1e55b7dd56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732634721511 2024-11-26T15:25:22,703 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting d09b93f70d3f4b5e947fb8bc9e75e2bb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732634721511 2024-11-26T15:25:22,718 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:22,719 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/89768a34ddfd47b3b8e22cf7e2266a1b is 50, key is test_row_0/A:col10/1732634721514/Put/seqid=0 2024-11-26T15:25:22,719 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-26T15:25:22,719 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:22,719 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-26T15:25:22,719 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:22,722 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:22,722 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/3a5d464753d74a00b756de9c0cef1b31 is 50, key is test_row_0/B:col10/1732634721514/Put/seqid=0 2024-11-26T15:25:22,723 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-26T15:25:22,723 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:22,725 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:22,725 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:22,725 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:22,726 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:22,726 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:22,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:22,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:22,726 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-26T15:25:22,726 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:22,726 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:22,726 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:22,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:22,726 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:22,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:22,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741937_1113 (size=13425) 2024-11-26T15:25:22,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741936_1112 (size=13425) 2024-11-26T15:25:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:22,807 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:22,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-26T15:25:22,808 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-26T15:25:22,808 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-26T15:25:22,808 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-26T15:25:22,808 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:22,808 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:22,808 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:22,808 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:22,808 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:22,808 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:22,808 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:22,808 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:22,808 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:22,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:22,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:22,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:22,809 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-26T15:25:22,809 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:22,809 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:22,810 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:22,810 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:22,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/0553804aa16b47b7a4a35ed3f37cde2d is 50, key is test_row_0/A:col10/1732634722804/Put/seqid=0 2024-11-26T15:25:22,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741938_1114 (size=12301) 2024-11-26T15:25:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T15:25:22,943 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:23,143 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/3a5d464753d74a00b756de9c0cef1b31 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/3a5d464753d74a00b756de9c0cef1b31 2024-11-26T15:25:23,144 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/89768a34ddfd47b3b8e22cf7e2266a1b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/89768a34ddfd47b3b8e22cf7e2266a1b 2024-11-26T15:25:23,150 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/B of 4c75b81af65e8ac430a5b8a152e75175 into 3a5d464753d74a00b756de9c0cef1b31(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:23,150 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:23,150 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/B, priority=13, startTime=1732634722698; duration=0sec 2024-11-26T15:25:23,150 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:23,150 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:B 2024-11-26T15:25:23,150 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:23,150 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/A of 4c75b81af65e8ac430a5b8a152e75175 into 89768a34ddfd47b3b8e22cf7e2266a1b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:23,150 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:23,150 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/A, priority=13, startTime=1732634722697; duration=0sec 2024-11-26T15:25:23,150 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:23,150 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:A 2024-11-26T15:25:23,151 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:23,151 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 4c75b81af65e8ac430a5b8a152e75175/C is initiating minor compaction (all files) 2024-11-26T15:25:23,152 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c75b81af65e8ac430a5b8a152e75175/C in TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:23,152 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/72befbf8b7c948b89d4bfe703d9b4823, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/b816948a14fa47cf828fde93e8ea90f1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/bc5e5bbef35a44e599765dde5859bf4e] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp, totalSize=37.0 K 2024-11-26T15:25:23,152 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 72befbf8b7c948b89d4bfe703d9b4823, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732634720183 2024-11-26T15:25:23,152 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting b816948a14fa47cf828fde93e8ea90f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1732634720340 2024-11-26T15:25:23,153 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting bc5e5bbef35a44e599765dde5859bf4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732634721511 2024-11-26T15:25:23,161 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c75b81af65e8ac430a5b8a152e75175#C#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:23,161 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/f661938846f743db9dcff1252cae94e0 is 50, key is test_row_0/C:col10/1732634721514/Put/seqid=0 2024-11-26T15:25:23,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741939_1115 (size=13425) 2024-11-26T15:25:23,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/0553804aa16b47b7a4a35ed3f37cde2d 2024-11-26T15:25:23,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/ab2e62cee0ba47009259b714d0e3cc04 is 50, key is test_row_0/B:col10/1732634722804/Put/seqid=0 2024-11-26T15:25:23,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741940_1116 (size=12301) 2024-11-26T15:25:23,573 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/f661938846f743db9dcff1252cae94e0 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/f661938846f743db9dcff1252cae94e0 2024-11-26T15:25:23,580 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c75b81af65e8ac430a5b8a152e75175/C of 4c75b81af65e8ac430a5b8a152e75175 into f661938846f743db9dcff1252cae94e0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:23,580 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:23,580 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., storeName=4c75b81af65e8ac430a5b8a152e75175/C, priority=13, startTime=1732634722698; duration=0sec 2024-11-26T15:25:23,580 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:23,580 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c75b81af65e8ac430a5b8a152e75175:C 2024-11-26T15:25:23,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/ab2e62cee0ba47009259b714d0e3cc04 2024-11-26T15:25:23,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/7800f60bd2694fcb86f224e57257415e is 50, key is test_row_0/C:col10/1732634722804/Put/seqid=0 2024-11-26T15:25:23,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741941_1117 (size=12301) 2024-11-26T15:25:24,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/7800f60bd2694fcb86f224e57257415e 2024-11-26T15:25:24,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/0553804aa16b47b7a4a35ed3f37cde2d as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/0553804aa16b47b7a4a35ed3f37cde2d 2024-11-26T15:25:24,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/0553804aa16b47b7a4a35ed3f37cde2d, entries=150, sequenceid=484, filesize=12.0 K 2024-11-26T15:25:24,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/ab2e62cee0ba47009259b714d0e3cc04 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/ab2e62cee0ba47009259b714d0e3cc04 2024-11-26T15:25:24,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/ab2e62cee0ba47009259b714d0e3cc04, entries=150, sequenceid=484, filesize=12.0 K 2024-11-26T15:25:24,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/7800f60bd2694fcb86f224e57257415e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/7800f60bd2694fcb86f224e57257415e 2024-11-26T15:25:24,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/7800f60bd2694fcb86f224e57257415e, entries=150, sequenceid=484, filesize=12.0 K 2024-11-26T15:25:24,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for 4c75b81af65e8ac430a5b8a152e75175 in 1282ms, sequenceid=484, compaction requested=false 2024-11-26T15:25:24,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c75b81af65e8ac430a5b8a152e75175: 2024-11-26T15:25:25,237 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T15:25:26,224 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-26T15:25:26,224 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:26,224 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:26,224 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:26,224 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:26,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T15:25:26,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-26T15:25:26,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-26T15:25:26,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-26T15:25:26,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-26T15:25:26,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-26T15:25:26,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T15:25:26,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2993 2024-11-26T15:25:26,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2987 2024-11-26T15:25:26,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T15:25:26,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1385 2024-11-26T15:25:26,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4135 rows 2024-11-26T15:25:26,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1392 2024-11-26T15:25:26,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4159 rows 2024-11-26T15:25:26,226 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T15:25:26,226 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T15:25:26,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:26,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:26,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:26,231 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T15:25:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T15:25:26,239 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634726238"}]},"ts":"1732634726238"} 2024-11-26T15:25:26,241 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T15:25:26,241 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T15:25:26,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T15:25:26,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c75b81af65e8ac430a5b8a152e75175, UNASSIGN}] 2024-11-26T15:25:26,247 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c75b81af65e8ac430a5b8a152e75175, UNASSIGN 2024-11-26T15:25:26,248 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=4c75b81af65e8ac430a5b8a152e75175, regionState=CLOSING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:26,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c75b81af65e8ac430a5b8a152e75175, UNASSIGN because future has completed 2024-11-26T15:25:26,251 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T15:25:26,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:25:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T15:25:26,412 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:26,412 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T15:25:26,413 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 4c75b81af65e8ac430a5b8a152e75175, disabling compactions & flushes 2024-11-26T15:25:26,413 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:26,413 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:26,413 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. after waiting 0 ms 2024-11-26T15:25:26,413 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:26,414 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 4c75b81af65e8ac430a5b8a152e75175 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-26T15:25:26,419 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/ce566ed45d8e4d99894994e45c98b8b4 is 50, key is test_row_1/A:col10/1732634726223/Put/seqid=0 2024-11-26T15:25:26,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741942_1118 (size=7415) 2024-11-26T15:25:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T15:25:26,825 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/ce566ed45d8e4d99894994e45c98b8b4 2024-11-26T15:25:26,841 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/dbed9a2660ea4dc097cc3e45e302def0 is 50, key is test_row_1/B:col10/1732634726223/Put/seqid=0 2024-11-26T15:25:26,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741943_1119 (size=7415) 2024-11-26T15:25:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T15:25:27,246 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/dbed9a2660ea4dc097cc3e45e302def0 2024-11-26T15:25:27,259 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/8ee90c83718247fcaf9ad026d9c1a33b is 50, key is test_row_1/C:col10/1732634726223/Put/seqid=0 2024-11-26T15:25:27,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741944_1120 (size=7415) 2024-11-26T15:25:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T15:25:27,665 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/8ee90c83718247fcaf9ad026d9c1a33b 2024-11-26T15:25:27,673 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/A/ce566ed45d8e4d99894994e45c98b8b4 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/ce566ed45d8e4d99894994e45c98b8b4 2024-11-26T15:25:27,680 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/ce566ed45d8e4d99894994e45c98b8b4, entries=50, sequenceid=491, filesize=7.2 K 2024-11-26T15:25:27,681 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/B/dbed9a2660ea4dc097cc3e45e302def0 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/dbed9a2660ea4dc097cc3e45e302def0 2024-11-26T15:25:27,687 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/dbed9a2660ea4dc097cc3e45e302def0, entries=50, sequenceid=491, filesize=7.2 K 2024-11-26T15:25:27,688 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/.tmp/C/8ee90c83718247fcaf9ad026d9c1a33b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/8ee90c83718247fcaf9ad026d9c1a33b 2024-11-26T15:25:27,693 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/8ee90c83718247fcaf9ad026d9c1a33b, entries=50, sequenceid=491, filesize=7.2 K 2024-11-26T15:25:27,694 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 4c75b81af65e8ac430a5b8a152e75175 in 1281ms, sequenceid=491, compaction requested=true 2024-11-26T15:25:27,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/3d8998b2ae1a4388ab95de90621b8418, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/c5b27d88a9f14b4b90797b5bfdda865d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/8eea54fc38784873959d4d3683053897, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/e653a5e891394351b919c08f3a1140b0, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b2981d99a06d487cacf5bb10f7a097de, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b5934c714e3444729ef972285ce74aea, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/cf367c1d6e1f4725873d1f65f4a1477b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/44e26d299e4246689290db787269edb5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5a931432ba5d4aa082dc95c1b3a89df5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/f2de7a7b5c0848109ffd5d0703b09815, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/966cb6a3ccbf4b07b41272d975d39f9b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/cef9c09625e8458c8592b2d2dfe3af8c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/ecf37df8a6da4cdc81cec1abfa500d3b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/602edad5a2554615be3fe0e32266e287, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/aea15a4b5f204179bf42486cf37938c9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5a8e35f9a9854021aca9c9276df66447, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/ae16636db9794f38a8ebabda2f3f2672, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/201a5ad9a4a54f238612e652bf2abd00, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/8a9c19de2f464d839198f3f6c9e928d5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/07fcfa50eae7476f8a782424e6aed748, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/e7a8420b29734cf1be9cf9f1268a106c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/039857bc9fae4e23afdf1c46e654267a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/c097310b5b30437d9cabd10a8c4e4a61, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b22ae60f8cda4281aab5d1b75af5d8ee, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/f7af43d7d08f4ffa81c879ef96b954fd, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/a5e1f38271c049a3806004da88db5d1b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/dcf99a29c8234a5caae16582cbbaa322, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/e10a27fe23204277a22ba71a69e56029, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/42d427cc1b1f4625bec51f46981f92a9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5c777c35e3324de18bf48d0a88a2344e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/74ef4197c680458cae87edaf04716cb5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/45fb0839161942cc9c22266f336e39aa, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/d09b93f70d3f4b5e947fb8bc9e75e2bb] to archive 2024-11-26T15:25:27,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:25:27,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/3d8998b2ae1a4388ab95de90621b8418 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/3d8998b2ae1a4388ab95de90621b8418 2024-11-26T15:25:27,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/c5b27d88a9f14b4b90797b5bfdda865d to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/c5b27d88a9f14b4b90797b5bfdda865d 2024-11-26T15:25:27,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/8eea54fc38784873959d4d3683053897 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/8eea54fc38784873959d4d3683053897 2024-11-26T15:25:27,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/e653a5e891394351b919c08f3a1140b0 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/e653a5e891394351b919c08f3a1140b0 2024-11-26T15:25:27,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b2981d99a06d487cacf5bb10f7a097de to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b2981d99a06d487cacf5bb10f7a097de 2024-11-26T15:25:27,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b5934c714e3444729ef972285ce74aea to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b5934c714e3444729ef972285ce74aea 2024-11-26T15:25:27,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/cf367c1d6e1f4725873d1f65f4a1477b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/cf367c1d6e1f4725873d1f65f4a1477b 2024-11-26T15:25:27,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/44e26d299e4246689290db787269edb5 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/44e26d299e4246689290db787269edb5 2024-11-26T15:25:27,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5a931432ba5d4aa082dc95c1b3a89df5 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5a931432ba5d4aa082dc95c1b3a89df5 2024-11-26T15:25:27,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/f2de7a7b5c0848109ffd5d0703b09815 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/f2de7a7b5c0848109ffd5d0703b09815 2024-11-26T15:25:27,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/966cb6a3ccbf4b07b41272d975d39f9b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/966cb6a3ccbf4b07b41272d975d39f9b 2024-11-26T15:25:27,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/cef9c09625e8458c8592b2d2dfe3af8c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/cef9c09625e8458c8592b2d2dfe3af8c 2024-11-26T15:25:27,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/ecf37df8a6da4cdc81cec1abfa500d3b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/ecf37df8a6da4cdc81cec1abfa500d3b 2024-11-26T15:25:27,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/602edad5a2554615be3fe0e32266e287 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/602edad5a2554615be3fe0e32266e287 2024-11-26T15:25:27,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/aea15a4b5f204179bf42486cf37938c9 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/aea15a4b5f204179bf42486cf37938c9 2024-11-26T15:25:27,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5a8e35f9a9854021aca9c9276df66447 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5a8e35f9a9854021aca9c9276df66447 2024-11-26T15:25:27,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/ae16636db9794f38a8ebabda2f3f2672 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/ae16636db9794f38a8ebabda2f3f2672 2024-11-26T15:25:27,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/201a5ad9a4a54f238612e652bf2abd00 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/201a5ad9a4a54f238612e652bf2abd00 2024-11-26T15:25:27,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/8a9c19de2f464d839198f3f6c9e928d5 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/8a9c19de2f464d839198f3f6c9e928d5 2024-11-26T15:25:27,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/07fcfa50eae7476f8a782424e6aed748 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/07fcfa50eae7476f8a782424e6aed748 2024-11-26T15:25:27,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/e7a8420b29734cf1be9cf9f1268a106c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/e7a8420b29734cf1be9cf9f1268a106c 2024-11-26T15:25:27,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/039857bc9fae4e23afdf1c46e654267a to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/039857bc9fae4e23afdf1c46e654267a 2024-11-26T15:25:27,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/c097310b5b30437d9cabd10a8c4e4a61 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/c097310b5b30437d9cabd10a8c4e4a61 2024-11-26T15:25:27,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b22ae60f8cda4281aab5d1b75af5d8ee to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/b22ae60f8cda4281aab5d1b75af5d8ee 2024-11-26T15:25:27,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/f7af43d7d08f4ffa81c879ef96b954fd to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/f7af43d7d08f4ffa81c879ef96b954fd 2024-11-26T15:25:27,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/a5e1f38271c049a3806004da88db5d1b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/a5e1f38271c049a3806004da88db5d1b 2024-11-26T15:25:27,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/dcf99a29c8234a5caae16582cbbaa322 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/dcf99a29c8234a5caae16582cbbaa322 2024-11-26T15:25:27,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/e10a27fe23204277a22ba71a69e56029 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/e10a27fe23204277a22ba71a69e56029 2024-11-26T15:25:27,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/42d427cc1b1f4625bec51f46981f92a9 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/42d427cc1b1f4625bec51f46981f92a9 2024-11-26T15:25:27,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5c777c35e3324de18bf48d0a88a2344e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/5c777c35e3324de18bf48d0a88a2344e 2024-11-26T15:25:27,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/74ef4197c680458cae87edaf04716cb5 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/74ef4197c680458cae87edaf04716cb5 2024-11-26T15:25:27,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/45fb0839161942cc9c22266f336e39aa to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/45fb0839161942cc9c22266f336e39aa 2024-11-26T15:25:27,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/d09b93f70d3f4b5e947fb8bc9e75e2bb to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/d09b93f70d3f4b5e947fb8bc9e75e2bb 2024-11-26T15:25:27,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/083ab5891bff46498c497c5cc58a40e8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/a55fdb4a53d74cab8526256a44ee5e86, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/825d96e2c5dc46058375f8ae8a06e986, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/6eb36aa9de694d5296d386fe6cb372d9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b5a94c7f78c14dd49f3aa2d376b1d0ed, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/7ce38b74b1174457baf875abd9ae6ad9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/342a87181cfa4768b8d05ff0d30db8f6, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/9a0848a5a6be4862a23aeb689d263302, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/a5c988e6889347eab143dc15cbae1e6f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b1b4943bab5a40debef771c593ca5bf1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/460f04fc22364a9ea9d3bc082403f4af, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/8df8a0d6a4ac40d8ac85b2336ab863fe, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b107a2e7ef414ae482b1c14c122fcce8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/53211272e7ea419186f4d53dd41c8ae6, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/20595fe2e36443adbfd624e3b7c81ef0, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0f350d9fa73a4612aded96e12511cb94, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/618535f94d934e44bcf59fadf3e91f62, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/9020a071ba2f464b9595a682b7d0430c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/ef865af10d5a4d1b9a58d2eccb8e5785, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0941b74ce7ce456ea6b8fb8a7bfd69c0, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1f9eaae662f443f595d4625d521d25bf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/7334346488db413d82946198fbe3b33a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1ef5c702c2264768a6154ebb5dd6db10, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0f8ea24d2a3a4a78863577475d045c39, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1f8d23b03e4a46d38172280ffa7c4b89, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/6533acc68b374f8d9b881d7bec6b5b7c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/ce3af76a359541b09568d930a5b0f239, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/e30bd4f5b29f4945834886de56254952, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/5dfe31b652ce4ff395a0e846bdc68adb, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/a35e5d66c7b74e5199b521818b26aa5c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/32d555a7efaa4f1a8eeccde024ab1965, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/142440c4927547d5a0da321a89c08f5f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/79dee34798474fcabb07ae1e55b7dd56] to archive 2024-11-26T15:25:27,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:25:27,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/083ab5891bff46498c497c5cc58a40e8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/083ab5891bff46498c497c5cc58a40e8 2024-11-26T15:25:27,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/a55fdb4a53d74cab8526256a44ee5e86 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/a55fdb4a53d74cab8526256a44ee5e86 2024-11-26T15:25:27,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/825d96e2c5dc46058375f8ae8a06e986 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/825d96e2c5dc46058375f8ae8a06e986 2024-11-26T15:25:27,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/6eb36aa9de694d5296d386fe6cb372d9 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/6eb36aa9de694d5296d386fe6cb372d9 2024-11-26T15:25:27,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b5a94c7f78c14dd49f3aa2d376b1d0ed to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b5a94c7f78c14dd49f3aa2d376b1d0ed 2024-11-26T15:25:27,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/7ce38b74b1174457baf875abd9ae6ad9 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/7ce38b74b1174457baf875abd9ae6ad9 2024-11-26T15:25:27,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/342a87181cfa4768b8d05ff0d30db8f6 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/342a87181cfa4768b8d05ff0d30db8f6 2024-11-26T15:25:27,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/9a0848a5a6be4862a23aeb689d263302 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/9a0848a5a6be4862a23aeb689d263302 2024-11-26T15:25:27,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/a5c988e6889347eab143dc15cbae1e6f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/a5c988e6889347eab143dc15cbae1e6f 2024-11-26T15:25:27,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b1b4943bab5a40debef771c593ca5bf1 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b1b4943bab5a40debef771c593ca5bf1 2024-11-26T15:25:27,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/460f04fc22364a9ea9d3bc082403f4af to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/460f04fc22364a9ea9d3bc082403f4af 2024-11-26T15:25:27,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/8df8a0d6a4ac40d8ac85b2336ab863fe to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/8df8a0d6a4ac40d8ac85b2336ab863fe 2024-11-26T15:25:27,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b107a2e7ef414ae482b1c14c122fcce8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/b107a2e7ef414ae482b1c14c122fcce8 2024-11-26T15:25:27,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/53211272e7ea419186f4d53dd41c8ae6 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/53211272e7ea419186f4d53dd41c8ae6 2024-11-26T15:25:27,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/20595fe2e36443adbfd624e3b7c81ef0 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/20595fe2e36443adbfd624e3b7c81ef0 2024-11-26T15:25:27,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0f350d9fa73a4612aded96e12511cb94 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0f350d9fa73a4612aded96e12511cb94 2024-11-26T15:25:27,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/618535f94d934e44bcf59fadf3e91f62 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/618535f94d934e44bcf59fadf3e91f62 2024-11-26T15:25:27,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/9020a071ba2f464b9595a682b7d0430c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/9020a071ba2f464b9595a682b7d0430c 2024-11-26T15:25:27,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/ef865af10d5a4d1b9a58d2eccb8e5785 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/ef865af10d5a4d1b9a58d2eccb8e5785 2024-11-26T15:25:27,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0941b74ce7ce456ea6b8fb8a7bfd69c0 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0941b74ce7ce456ea6b8fb8a7bfd69c0 2024-11-26T15:25:27,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1f9eaae662f443f595d4625d521d25bf to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1f9eaae662f443f595d4625d521d25bf 2024-11-26T15:25:27,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/7334346488db413d82946198fbe3b33a to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/7334346488db413d82946198fbe3b33a 2024-11-26T15:25:27,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1ef5c702c2264768a6154ebb5dd6db10 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1ef5c702c2264768a6154ebb5dd6db10 2024-11-26T15:25:27,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0f8ea24d2a3a4a78863577475d045c39 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/0f8ea24d2a3a4a78863577475d045c39 2024-11-26T15:25:27,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1f8d23b03e4a46d38172280ffa7c4b89 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/1f8d23b03e4a46d38172280ffa7c4b89 2024-11-26T15:25:27,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/6533acc68b374f8d9b881d7bec6b5b7c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/6533acc68b374f8d9b881d7bec6b5b7c 2024-11-26T15:25:27,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/ce3af76a359541b09568d930a5b0f239 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/ce3af76a359541b09568d930a5b0f239 2024-11-26T15:25:27,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/e30bd4f5b29f4945834886de56254952 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/e30bd4f5b29f4945834886de56254952 2024-11-26T15:25:27,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/5dfe31b652ce4ff395a0e846bdc68adb to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/5dfe31b652ce4ff395a0e846bdc68adb 2024-11-26T15:25:27,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/a35e5d66c7b74e5199b521818b26aa5c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/a35e5d66c7b74e5199b521818b26aa5c 2024-11-26T15:25:27,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/32d555a7efaa4f1a8eeccde024ab1965 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/32d555a7efaa4f1a8eeccde024ab1965 2024-11-26T15:25:27,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/142440c4927547d5a0da321a89c08f5f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/142440c4927547d5a0da321a89c08f5f 2024-11-26T15:25:27,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/79dee34798474fcabb07ae1e55b7dd56 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/79dee34798474fcabb07ae1e55b7dd56 2024-11-26T15:25:27,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/8b203e0193e74666a2a309aeafbf5ebf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/191405d332c54c11a86a59150845f7c5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/9728c409170b4432bd0eabef273f1179, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/0d3d01b3496e45f9b7c60ba0216bf498, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/271e9e7937074d0b82550f121ac4a5b3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/bd5d1ff10db247de9fc49bc7ec5bcaf8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/93346c1d3a8e4723a0678110416e9227, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/41345d97520b41f98e431298e6efb117, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/59d359a4b7444842927ca2fa9f3178ee, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/d7abe8e46905409c8489b83dd3247985, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/ddce9c9e25d6471597f957fd1fe74f6a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/2337cc20c3db4088bdbf167569197172, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4b08bfd5ac9a45e09609f772849cbacc, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/96e76b1bcf3d4b629a7456cbe95f6363, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/98e6c39193e84ed4a6d04692beef66fd, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/667ff3bcd977452889fe4a27431b99fc, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/9327b0e442334c0aae679b44521b3351, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/06fb766588f9436d8965c2f68b696ff8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/d307ed3a4fe24b928c60c7998b1f773b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/a3bb1e7065cd494bb0635298fa0ac363, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4797669add244d568f36ccd784914468, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4d3535166eb04bf6a0d23ba0c023db8f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/88677048548241b1b5642102a46429e1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/3d66adb7c70345cf82a97e79d98476ab, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/5f6cd0ad11ac4c2bb62a0e4f21febb9c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/90cf0566a7164d4cbe6ddb9cb245c454, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/265cef5c0fe040c3a302704767a22f1e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/c81d425850904caf855d1d4915995534, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/e0fe81cd6c8540ab9afd88fc9063ce29, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/72befbf8b7c948b89d4bfe703d9b4823, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/a53078025da645a3b9fddf4afde228f1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/b816948a14fa47cf828fde93e8ea90f1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/bc5e5bbef35a44e599765dde5859bf4e] to archive 2024-11-26T15:25:27,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:25:27,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/8b203e0193e74666a2a309aeafbf5ebf to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/8b203e0193e74666a2a309aeafbf5ebf 2024-11-26T15:25:27,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/191405d332c54c11a86a59150845f7c5 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/191405d332c54c11a86a59150845f7c5 2024-11-26T15:25:27,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/9728c409170b4432bd0eabef273f1179 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/9728c409170b4432bd0eabef273f1179 2024-11-26T15:25:27,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/0d3d01b3496e45f9b7c60ba0216bf498 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/0d3d01b3496e45f9b7c60ba0216bf498 2024-11-26T15:25:27,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/271e9e7937074d0b82550f121ac4a5b3 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/271e9e7937074d0b82550f121ac4a5b3 2024-11-26T15:25:27,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/bd5d1ff10db247de9fc49bc7ec5bcaf8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/bd5d1ff10db247de9fc49bc7ec5bcaf8 2024-11-26T15:25:27,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/93346c1d3a8e4723a0678110416e9227 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/93346c1d3a8e4723a0678110416e9227 2024-11-26T15:25:27,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/41345d97520b41f98e431298e6efb117 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/41345d97520b41f98e431298e6efb117 2024-11-26T15:25:27,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/59d359a4b7444842927ca2fa9f3178ee to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/59d359a4b7444842927ca2fa9f3178ee 2024-11-26T15:25:27,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/d7abe8e46905409c8489b83dd3247985 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/d7abe8e46905409c8489b83dd3247985 2024-11-26T15:25:27,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/ddce9c9e25d6471597f957fd1fe74f6a to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/ddce9c9e25d6471597f957fd1fe74f6a 2024-11-26T15:25:27,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/2337cc20c3db4088bdbf167569197172 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/2337cc20c3db4088bdbf167569197172 2024-11-26T15:25:27,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4b08bfd5ac9a45e09609f772849cbacc to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4b08bfd5ac9a45e09609f772849cbacc 2024-11-26T15:25:27,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/96e76b1bcf3d4b629a7456cbe95f6363 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/96e76b1bcf3d4b629a7456cbe95f6363 2024-11-26T15:25:27,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/98e6c39193e84ed4a6d04692beef66fd to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/98e6c39193e84ed4a6d04692beef66fd 2024-11-26T15:25:27,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/667ff3bcd977452889fe4a27431b99fc to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/667ff3bcd977452889fe4a27431b99fc 2024-11-26T15:25:27,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/9327b0e442334c0aae679b44521b3351 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/9327b0e442334c0aae679b44521b3351 2024-11-26T15:25:27,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/06fb766588f9436d8965c2f68b696ff8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/06fb766588f9436d8965c2f68b696ff8 2024-11-26T15:25:27,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/d307ed3a4fe24b928c60c7998b1f773b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/d307ed3a4fe24b928c60c7998b1f773b 2024-11-26T15:25:27,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/a3bb1e7065cd494bb0635298fa0ac363 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/a3bb1e7065cd494bb0635298fa0ac363 2024-11-26T15:25:27,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4797669add244d568f36ccd784914468 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4797669add244d568f36ccd784914468 2024-11-26T15:25:27,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4d3535166eb04bf6a0d23ba0c023db8f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/4d3535166eb04bf6a0d23ba0c023db8f 2024-11-26T15:25:27,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/88677048548241b1b5642102a46429e1 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/88677048548241b1b5642102a46429e1 2024-11-26T15:25:27,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/3d66adb7c70345cf82a97e79d98476ab to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/3d66adb7c70345cf82a97e79d98476ab 2024-11-26T15:25:27,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/5f6cd0ad11ac4c2bb62a0e4f21febb9c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/5f6cd0ad11ac4c2bb62a0e4f21febb9c 2024-11-26T15:25:27,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/90cf0566a7164d4cbe6ddb9cb245c454 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/90cf0566a7164d4cbe6ddb9cb245c454 2024-11-26T15:25:27,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/265cef5c0fe040c3a302704767a22f1e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/265cef5c0fe040c3a302704767a22f1e 2024-11-26T15:25:27,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/c81d425850904caf855d1d4915995534 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/c81d425850904caf855d1d4915995534 2024-11-26T15:25:27,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/e0fe81cd6c8540ab9afd88fc9063ce29 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/e0fe81cd6c8540ab9afd88fc9063ce29 2024-11-26T15:25:27,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/72befbf8b7c948b89d4bfe703d9b4823 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/72befbf8b7c948b89d4bfe703d9b4823 2024-11-26T15:25:27,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/a53078025da645a3b9fddf4afde228f1 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/a53078025da645a3b9fddf4afde228f1 2024-11-26T15:25:27,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/b816948a14fa47cf828fde93e8ea90f1 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/b816948a14fa47cf828fde93e8ea90f1 2024-11-26T15:25:27,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/bc5e5bbef35a44e599765dde5859bf4e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/bc5e5bbef35a44e599765dde5859bf4e 2024-11-26T15:25:27,840 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/recovered.edits/494.seqid, newMaxSeqId=494, maxSeqId=1 2024-11-26T15:25:27,842 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. 2024-11-26T15:25:27,843 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 4c75b81af65e8ac430a5b8a152e75175: Waiting for close lock at 1732634726413Running coprocessor pre-close hooks at 1732634726413Disabling compacts and flushes for region at 1732634726413Disabling writes for close at 1732634726413Obtaining lock to block concurrent updates at 1732634726414 (+1 ms)Preparing flush snapshotting stores in 4c75b81af65e8ac430a5b8a152e75175 at 1732634726414Finished memstore snapshotting TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1732634726414Flushing stores of TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175. at 1732634726415 (+1 ms)Flushing 4c75b81af65e8ac430a5b8a152e75175/A: creating writer at 1732634726415Flushing 4c75b81af65e8ac430a5b8a152e75175/A: appending metadata at 1732634726418 (+3 ms)Flushing 4c75b81af65e8ac430a5b8a152e75175/A: closing flushed file at 1732634726418Flushing 4c75b81af65e8ac430a5b8a152e75175/B: creating writer at 1732634726836 (+418 ms)Flushing 4c75b81af65e8ac430a5b8a152e75175/B: appending metadata at 1732634726840 (+4 ms)Flushing 4c75b81af65e8ac430a5b8a152e75175/B: closing flushed file at 1732634726840Flushing 4c75b81af65e8ac430a5b8a152e75175/C: creating writer at 1732634727255 (+415 ms)Flushing 4c75b81af65e8ac430a5b8a152e75175/C: appending metadata at 1732634727258 (+3 ms)Flushing 4c75b81af65e8ac430a5b8a152e75175/C: closing flushed file at 1732634727259 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24374b44: reopening flushed file at 1732634727672 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25e5239b: reopening flushed file at 1732634727680 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f6841f3: reopening flushed file at 1732634727687 (+7 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 4c75b81af65e8ac430a5b8a152e75175 in 1281ms, sequenceid=491, compaction requested=true at 1732634727694 (+7 ms)Writing region close event to WAL at 1732634727836 (+142 ms)Running coprocessor post-close hooks at 1732634727840 (+4 ms)Closed at 1732634727842 (+2 ms) 2024-11-26T15:25:27,846 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:27,846 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=4c75b81af65e8ac430a5b8a152e75175, regionState=CLOSED 2024-11-26T15:25:27,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:25:27,851 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-26T15:25:27,851 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c75b81af65e8ac430a5b8a152e75175, server=6cc2bc8bf733,45729,1732634697146 in 1.5990 sec 2024-11-26T15:25:27,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-26T15:25:27,853 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c75b81af65e8ac430a5b8a152e75175, UNASSIGN in 1.6050 sec 2024-11-26T15:25:27,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-26T15:25:27,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6120 sec 2024-11-26T15:25:27,857 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634727857"}]},"ts":"1732634727857"} 2024-11-26T15:25:27,859 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T15:25:27,859 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T15:25:27,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6260 sec 2024-11-26T15:25:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T15:25:28,383 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:28,386 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T15:25:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:28,391 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-26T15:25:28,393 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:28,397 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:28,400 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/recovered.edits] 2024-11-26T15:25:28,404 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/0553804aa16b47b7a4a35ed3f37cde2d to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/0553804aa16b47b7a4a35ed3f37cde2d 2024-11-26T15:25:28,406 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/89768a34ddfd47b3b8e22cf7e2266a1b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/89768a34ddfd47b3b8e22cf7e2266a1b 2024-11-26T15:25:28,408 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/ce566ed45d8e4d99894994e45c98b8b4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/A/ce566ed45d8e4d99894994e45c98b8b4 2024-11-26T15:25:28,411 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/3a5d464753d74a00b756de9c0cef1b31 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/3a5d464753d74a00b756de9c0cef1b31 2024-11-26T15:25:28,413 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/ab2e62cee0ba47009259b714d0e3cc04 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/ab2e62cee0ba47009259b714d0e3cc04 2024-11-26T15:25:28,414 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/dbed9a2660ea4dc097cc3e45e302def0 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/B/dbed9a2660ea4dc097cc3e45e302def0 2024-11-26T15:25:28,417 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/7800f60bd2694fcb86f224e57257415e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/7800f60bd2694fcb86f224e57257415e 2024-11-26T15:25:28,421 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/8ee90c83718247fcaf9ad026d9c1a33b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/8ee90c83718247fcaf9ad026d9c1a33b 2024-11-26T15:25:28,423 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/f661938846f743db9dcff1252cae94e0 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/C/f661938846f743db9dcff1252cae94e0 2024-11-26T15:25:28,427 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/recovered.edits/494.seqid to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175/recovered.edits/494.seqid 2024-11-26T15:25:28,428 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/4c75b81af65e8ac430a5b8a152e75175 2024-11-26T15:25:28,428 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T15:25:28,434 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:28,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-26T15:25:28,444 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T15:25:28,479 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T15:25:28,481 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:28,481 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T15:25:28,481 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732634728481"}]},"ts":"9223372036854775807"} 2024-11-26T15:25:28,486 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T15:25:28,486 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4c75b81af65e8ac430a5b8a152e75175, NAME => 'TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T15:25:28,486 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T15:25:28,487 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732634728486"}]},"ts":"9223372036854775807"} 2024-11-26T15:25:28,490 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T15:25:28,491 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:28,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 104 msec 2024-11-26T15:25:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-26T15:25:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T15:25:28,504 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:28,517 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_869531393_22 at /127.0.0.1:54900 [Waiting for operation #352] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-908112358_22 at /127.0.0.1:56766 [Waiting for operation #317] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;6cc2bc8bf733:45729-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=753 (was 710) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9464 (was 6891) - AvailableMemoryMB LEAK? - 2024-11-26T15:25:28,527 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=753, ProcessCount=11, AvailableMemoryMB=9464 2024-11-26T15:25:28,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T15:25:28,529 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T15:25:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:28,532 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T15:25:28,532 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:28,532 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-26T15:25:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-26T15:25:28,533 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T15:25:28,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741945_1121 (size=959) 2024-11-26T15:25:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-26T15:25:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-26T15:25:28,943 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6 2024-11-26T15:25:28,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741946_1122 (size=53) 2024-11-26T15:25:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-26T15:25:29,351 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:25:29,351 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 997d8d2a525d3ba7cd152ef78c94cfd4, disabling compactions & flushes 2024-11-26T15:25:29,351 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:29,351 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:29,351 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. after waiting 0 ms 2024-11-26T15:25:29,351 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:29,351 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:29,351 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 997d8d2a525d3ba7cd152ef78c94cfd4: Waiting for close lock at 1732634729351Disabling compacts and flushes for region at 1732634729351Disabling writes for close at 1732634729351Writing region close event to WAL at 1732634729351Closed at 1732634729351 2024-11-26T15:25:29,353 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T15:25:29,353 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732634729353"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732634729353"}]},"ts":"1732634729353"} 2024-11-26T15:25:29,355 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T15:25:29,356 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T15:25:29,356 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634729356"}]},"ts":"1732634729356"} 2024-11-26T15:25:29,358 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T15:25:29,359 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=997d8d2a525d3ba7cd152ef78c94cfd4, ASSIGN}] 2024-11-26T15:25:29,360 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=997d8d2a525d3ba7cd152ef78c94cfd4, ASSIGN 2024-11-26T15:25:29,360 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=997d8d2a525d3ba7cd152ef78c94cfd4, ASSIGN; state=OFFLINE, location=6cc2bc8bf733,45729,1732634697146; forceNewPlan=false, retain=false 2024-11-26T15:25:29,511 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=997d8d2a525d3ba7cd152ef78c94cfd4, regionState=OPENING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:29,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=997d8d2a525d3ba7cd152ef78c94cfd4, ASSIGN because future has completed 2024-11-26T15:25:29,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:25:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-26T15:25:29,674 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:29,674 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:25:29,674 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:29,674 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:25:29,675 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:29,675 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:29,677 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:29,681 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 997d8d2a525d3ba7cd152ef78c94cfd4 columnFamilyName A 2024-11-26T15:25:29,681 DEBUG [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:29,682 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] regionserver.HStore(327): Store=997d8d2a525d3ba7cd152ef78c94cfd4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:25:29,682 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:29,685 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 997d8d2a525d3ba7cd152ef78c94cfd4 columnFamilyName B 2024-11-26T15:25:29,685 DEBUG [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:29,686 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] regionserver.HStore(327): Store=997d8d2a525d3ba7cd152ef78c94cfd4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:25:29,686 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:29,688 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 997d8d2a525d3ba7cd152ef78c94cfd4 columnFamilyName C 2024-11-26T15:25:29,688 DEBUG [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:29,689 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] regionserver.HStore(327): Store=997d8d2a525d3ba7cd152ef78c94cfd4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:25:29,689 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:29,690 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:29,690 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:29,691 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:29,691 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:29,692 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T15:25:29,693 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:29,696 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:25:29,696 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 997d8d2a525d3ba7cd152ef78c94cfd4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65457676, jitterRate=-0.024604618549346924}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T15:25:29,696 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:29,697 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 997d8d2a525d3ba7cd152ef78c94cfd4: Running coprocessor pre-open hook at 1732634729675Writing region info on filesystem at 1732634729675Initializing all the Stores at 1732634729676 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634729676Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634729677 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634729677Cleaning up temporary data from old regions at 1732634729691 (+14 ms)Running coprocessor post-open hooks at 1732634729696 (+5 ms)Region opened successfully at 1732634729697 (+1 ms) 2024-11-26T15:25:29,698 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., pid=30, masterSystemTime=1732634729669 2024-11-26T15:25:29,700 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:29,700 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:29,701 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=997d8d2a525d3ba7cd152ef78c94cfd4, regionState=OPEN, openSeqNum=2, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:29,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:25:29,705 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-26T15:25:29,705 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 in 187 msec 2024-11-26T15:25:29,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-26T15:25:29,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=997d8d2a525d3ba7cd152ef78c94cfd4, ASSIGN in 347 msec 2024-11-26T15:25:29,707 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T15:25:29,708 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634729707"}]},"ts":"1732634729707"} 2024-11-26T15:25:29,709 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T15:25:29,710 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T15:25:29,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1810 sec 2024-11-26T15:25:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-26T15:25:30,684 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:30,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594c9324, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:30,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:30,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:30,688 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@412f659e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:30,693 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d4ac981, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:30,695 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:30,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:30,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:30,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:30,701 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T15:25:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 6cc2bc8bf733,37081,1732634696319 2024-11-26T15:25:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@28a8f79a 2024-11-26T15:25:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T15:25:30,703 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T15:25:30,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T15:25:30,709 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T15:25:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:30,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741947_1123 (size=995) 2024-11-26T15:25:31,130 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-26T15:25:31,131 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-26T15:25:31,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T15:25:31,145 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=997d8d2a525d3ba7cd152ef78c94cfd4, REOPEN/MOVE}] 2024-11-26T15:25:31,146 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=997d8d2a525d3ba7cd152ef78c94cfd4, REOPEN/MOVE 2024-11-26T15:25:31,147 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=997d8d2a525d3ba7cd152ef78c94cfd4, regionState=CLOSING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:31,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=997d8d2a525d3ba7cd152ef78c94cfd4, REOPEN/MOVE because future has completed 2024-11-26T15:25:31,149 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T15:25:31,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:25:31,301 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,301 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T15:25:31,302 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 997d8d2a525d3ba7cd152ef78c94cfd4, disabling compactions & flushes 2024-11-26T15:25:31,302 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:31,302 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:31,302 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. after waiting 0 ms 2024-11-26T15:25:31,302 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:31,349 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T15:25:31,350 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:31,350 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 997d8d2a525d3ba7cd152ef78c94cfd4: Waiting for close lock at 1732634731302Running coprocessor pre-close hooks at 1732634731302Disabling compacts and flushes for region at 1732634731302Disabling writes for close at 1732634731302Writing region close event to WAL at 1732634731303 (+1 ms)Running coprocessor post-close hooks at 1732634731350 (+47 ms)Closed at 1732634731350 2024-11-26T15:25:31,350 WARN [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 997d8d2a525d3ba7cd152ef78c94cfd4 to self. 2024-11-26T15:25:31,352 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,353 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=997d8d2a525d3ba7cd152ef78c94cfd4, regionState=CLOSED 2024-11-26T15:25:31,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:25:31,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-26T15:25:31,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 in 206 msec 2024-11-26T15:25:31,358 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=997d8d2a525d3ba7cd152ef78c94cfd4, REOPEN/MOVE; state=CLOSED, location=6cc2bc8bf733,45729,1732634697146; forceNewPlan=false, retain=true 2024-11-26T15:25:31,509 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=997d8d2a525d3ba7cd152ef78c94cfd4, regionState=OPENING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:31,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=997d8d2a525d3ba7cd152ef78c94cfd4, REOPEN/MOVE because future has completed 2024-11-26T15:25:31,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:25:31,668 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:31,669 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:25:31,669 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,669 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:25:31,670 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,670 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,676 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,683 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 997d8d2a525d3ba7cd152ef78c94cfd4 columnFamilyName A 2024-11-26T15:25:31,684 DEBUG [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:31,685 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] regionserver.HStore(327): Store=997d8d2a525d3ba7cd152ef78c94cfd4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:25:31,685 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,686 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 997d8d2a525d3ba7cd152ef78c94cfd4 columnFamilyName B 2024-11-26T15:25:31,686 DEBUG [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:31,687 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] regionserver.HStore(327): Store=997d8d2a525d3ba7cd152ef78c94cfd4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:25:31,687 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,688 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 997d8d2a525d3ba7cd152ef78c94cfd4 columnFamilyName C 2024-11-26T15:25:31,688 DEBUG [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:31,688 INFO [StoreOpener-997d8d2a525d3ba7cd152ef78c94cfd4-1 {}] regionserver.HStore(327): Store=997d8d2a525d3ba7cd152ef78c94cfd4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:25:31,688 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,689 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,690 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,691 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,691 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,692 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T15:25:31,693 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,693 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 997d8d2a525d3ba7cd152ef78c94cfd4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72718627, jitterRate=0.08359198272228241}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T15:25:31,693 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,695 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 997d8d2a525d3ba7cd152ef78c94cfd4: Running coprocessor pre-open hook at 1732634731670Writing region info on filesystem at 1732634731670Initializing all the Stores at 1732634731671 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634731671Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634731672 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634731672Cleaning up temporary data from old regions at 1732634731691 (+19 ms)Running coprocessor post-open hooks at 1732634731693 (+2 ms)Region opened successfully at 1732634731695 (+2 ms) 2024-11-26T15:25:31,695 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., pid=35, masterSystemTime=1732634731663 2024-11-26T15:25:31,697 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:31,697 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:31,698 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=997d8d2a525d3ba7cd152ef78c94cfd4, regionState=OPEN, openSeqNum=5, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:31,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:25:31,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-26T15:25:31,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 in 189 msec 2024-11-26T15:25:31,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-26T15:25:31,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=997d8d2a525d3ba7cd152ef78c94cfd4, REOPEN/MOVE in 556 msec 2024-11-26T15:25:31,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-26T15:25:31,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 566 msec 2024-11-26T15:25:31,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 995 msec 2024-11-26T15:25:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-26T15:25:31,710 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:31,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fcb20d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,711 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:31,711 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:31,712 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3546078f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:31,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:31,713 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39dddd7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e853879, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:31,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:31,715 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e5ba841, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:31,717 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:31,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ee7f00f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a00ea56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,718 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:31,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:31,719 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65807d0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:31,720 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:31,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67d150c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636d090d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:31,721 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:31,722 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:31,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:31,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:31,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379774a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:31,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:31,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:31,724 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8bbdc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e0133ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:31,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:31,726 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aca91e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:31,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:31,727 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61183aed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc5db83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,728 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:31,729 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:31,729 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99936c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:31,731 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5be0e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4870d163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:31,732 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:31,733 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23adfa77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:31,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:31,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:31,735 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33babeb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53e9634b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,736 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:31,736 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:31,737 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392aeb4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:31,738 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:31,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b13c971, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f7e914, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,739 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:25:31,739 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:25:31,740 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:25:31,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:25:31,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:25:31,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ac2f62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:25:31,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:25:31,741 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:31,741 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:25:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e2ad28e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:25:31,743 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:25:31,743 DEBUG [Thread-630 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:31,743 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:31,743 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:31,744 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:31,744 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:31,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-26T15:25:31,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:31,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:31,744 DEBUG [Thread-628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:31,745 DEBUG [Thread-628 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:31,745 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:25:31,745 DEBUG [Thread-628 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T15:25:31,746 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:25:31,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:25:31,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:31,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:31,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:31,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:31,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:31,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:31,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:25:31,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:25:31,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:25:31,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:25:31,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:25:31,753 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:31,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:31,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:31,755 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:25:31,755 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:25:31,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:31,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:25:31,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:31,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:25:31,757 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:31,757 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:31,757 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:25:31,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:25:31,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:25:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,763 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:31,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T15:25:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:25:31,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d660e34f41ca4bb096c93d549f7f8156_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634731752/Put/seqid=0 2024-11-26T15:25:31,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741948_1124 (size=12154) 2024-11-26T15:25:31,822 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:31,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634791813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:31,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634791813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:31,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634791815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:31,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:31,828 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d660e34f41ca4bb096c93d549f7f8156_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d660e34f41ca4bb096c93d549f7f8156_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:31,829 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/66ae5c4df92944fca22cbab39adfd4e4, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:31,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634791825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:31,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:31,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:31,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634791825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:31,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:31,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/66ae5c4df92944fca22cbab39adfd4e4 is 175, key is test_row_0/A:col10/1732634731752/Put/seqid=0 2024-11-26T15:25:31,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741949_1125 (size=30955) 2024-11-26T15:25:31,849 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/66ae5c4df92944fca22cbab39adfd4e4 2024-11-26T15:25:31,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T15:25:31,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/ce27236042234386a59319b3abc6d9ef is 50, key is test_row_0/B:col10/1732634731752/Put/seqid=0 2024-11-26T15:25:31,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T15:25:31,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:31,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:31,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:31,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:31,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:31,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741950_1126 (size=12001) 2024-11-26T15:25:31,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:31,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634791933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:31,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:31,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634791933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:31,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:31,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:31,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:31,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634791933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:31,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:31,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634791943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:31,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634791948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T15:25:32,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:32,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:32,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:32,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T15:25:32,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634792143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634792144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634792143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634792163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634792172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T15:25:32,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:32,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:32,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:32,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/ce27236042234386a59319b3abc6d9ef 2024-11-26T15:25:32,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/60b68554821e4cc198852cf5a5e4e295 is 50, key is test_row_0/C:col10/1732634731752/Put/seqid=0 2024-11-26T15:25:32,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T15:25:32,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:32,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:32,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:32,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T15:25:32,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741951_1127 (size=12001) 2024-11-26T15:25:32,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/60b68554821e4cc198852cf5a5e4e295 2024-11-26T15:25:32,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/66ae5c4df92944fca22cbab39adfd4e4 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/66ae5c4df92944fca22cbab39adfd4e4 2024-11-26T15:25:32,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/66ae5c4df92944fca22cbab39adfd4e4, entries=150, sequenceid=16, filesize=30.2 K 2024-11-26T15:25:32,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/ce27236042234386a59319b3abc6d9ef as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/ce27236042234386a59319b3abc6d9ef 2024-11-26T15:25:32,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/ce27236042234386a59319b3abc6d9ef, entries=150, sequenceid=16, filesize=11.7 K 2024-11-26T15:25:32,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/60b68554821e4cc198852cf5a5e4e295 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/60b68554821e4cc198852cf5a5e4e295 2024-11-26T15:25:32,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/60b68554821e4cc198852cf5a5e4e295, entries=150, sequenceid=16, filesize=11.7 K 2024-11-26T15:25:32,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 659ms, sequenceid=16, compaction requested=false 2024-11-26T15:25:32,423 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T15:25:32,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:32,459 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:32,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-26T15:25:32,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260aa07df34abd4cbe8d8b065d6a5d53ea_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634731819/Put/seqid=0 2024-11-26T15:25:32,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741952_1128 (size=14594) 2024-11-26T15:25:32,495 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:32,503 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260aa07df34abd4cbe8d8b065d6a5d53ea_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260aa07df34abd4cbe8d8b065d6a5d53ea_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:32,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634792483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634792483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634792485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,506 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/de18ba24690b483395e789038c0e68cb, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:32,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/de18ba24690b483395e789038c0e68cb is 175, key is test_row_0/A:col10/1732634731819/Put/seqid=0 2024-11-26T15:25:32,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741953_1129 (size=39549) 2024-11-26T15:25:32,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634792501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634792504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,518 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/de18ba24690b483395e789038c0e68cb 2024-11-26T15:25:32,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T15:25:32,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:32,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:32,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:32,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/6201e4fa90104fc7819b6f7c3cc3d7c4 is 50, key is test_row_0/B:col10/1732634731819/Put/seqid=0 2024-11-26T15:25:32,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741954_1130 (size=12001) 2024-11-26T15:25:32,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/6201e4fa90104fc7819b6f7c3cc3d7c4 2024-11-26T15:25:32,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/8caaf450f3c54d6591b0e56d1c166235 is 50, key is test_row_0/C:col10/1732634731819/Put/seqid=0 2024-11-26T15:25:32,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741955_1131 (size=12001) 2024-11-26T15:25:32,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634792613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634792613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634792623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634792623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T15:25:32,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:32,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:32,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:32,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634792823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634792823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634792833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634792833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:32,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:32,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T15:25:32,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:32,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:32,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:32,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T15:25:32,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/8caaf450f3c54d6591b0e56d1c166235 2024-11-26T15:25:32,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T15:25:32,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:32,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:32,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:32,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:33,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/de18ba24690b483395e789038c0e68cb as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/de18ba24690b483395e789038c0e68cb 2024-11-26T15:25:33,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/de18ba24690b483395e789038c0e68cb, entries=200, sequenceid=42, filesize=38.6 K 2024-11-26T15:25:33,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/6201e4fa90104fc7819b6f7c3cc3d7c4 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/6201e4fa90104fc7819b6f7c3cc3d7c4 2024-11-26T15:25:33,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634793012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/6201e4fa90104fc7819b6f7c3cc3d7c4, entries=150, sequenceid=42, filesize=11.7 K 2024-11-26T15:25:33,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/8caaf450f3c54d6591b0e56d1c166235 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/8caaf450f3c54d6591b0e56d1c166235 2024-11-26T15:25:33,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/8caaf450f3c54d6591b0e56d1c166235, entries=150, sequenceid=42, filesize=11.7 K 2024-11-26T15:25:33,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 585ms, sequenceid=42, compaction requested=false 2024-11-26T15:25:33,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:33,135 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:33,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T15:25:33,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T15:25:33,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:33,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:33,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:33,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:33,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:33,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:33,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b488f85fe9b143a1bf52f439e51d64af_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634732460/Put/seqid=0 2024-11-26T15:25:33,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741956_1132 (size=17034) 2024-11-26T15:25:33,192 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:33,199 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b488f85fe9b143a1bf52f439e51d64af_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b488f85fe9b143a1bf52f439e51d64af_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:33,201 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/26e17c901b9b4171be0b169241d3e08d, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:33,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/26e17c901b9b4171be0b169241d3e08d is 175, key is test_row_0/A:col10/1732634732460/Put/seqid=0 2024-11-26T15:25:33,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634793198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634793200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634793207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634793208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741957_1133 (size=48139) 2024-11-26T15:25:33,242 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/26e17c901b9b4171be0b169241d3e08d 2024-11-26T15:25:33,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/b6e1bf7be74746baae5f5bd9c70cd373 is 50, key is test_row_0/B:col10/1732634732460/Put/seqid=0 2024-11-26T15:25:33,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741958_1134 (size=12001) 2024-11-26T15:25:33,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T15:25:33,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:33,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:33,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:33,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:33,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:33,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:33,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634793313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634793313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634793323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634793323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,384 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T15:25:33,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T15:25:33,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:33,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:33,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:33,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:33,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:33,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:33,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634793523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634793532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634793532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634793532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T15:25:33,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:33,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:33,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:33,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:33,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:33,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/b6e1bf7be74746baae5f5bd9c70cd373 2024-11-26T15:25:33,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/72efeee9270347f1b94f898660157f38 is 50, key is test_row_0/C:col10/1732634732460/Put/seqid=0 2024-11-26T15:25:33,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741959_1135 (size=12001) 2024-11-26T15:25:33,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/72efeee9270347f1b94f898660157f38 2024-11-26T15:25:33,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/26e17c901b9b4171be0b169241d3e08d as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/26e17c901b9b4171be0b169241d3e08d 2024-11-26T15:25:33,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/26e17c901b9b4171be0b169241d3e08d, entries=250, sequenceid=54, filesize=47.0 K 2024-11-26T15:25:33,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/b6e1bf7be74746baae5f5bd9c70cd373 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/b6e1bf7be74746baae5f5bd9c70cd373 2024-11-26T15:25:33,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/b6e1bf7be74746baae5f5bd9c70cd373, entries=150, sequenceid=54, filesize=11.7 K 2024-11-26T15:25:33,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/72efeee9270347f1b94f898660157f38 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/72efeee9270347f1b94f898660157f38 2024-11-26T15:25:33,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/72efeee9270347f1b94f898660157f38, entries=150, sequenceid=54, filesize=11.7 K 2024-11-26T15:25:33,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 589ms, sequenceid=54, compaction requested=true 2024-11-26T15:25:33,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:33,725 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:33,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:33,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:33,725 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:33,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:33,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:33,726 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:33,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:33,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:33,726 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/A is initiating minor compaction (all files) 2024-11-26T15:25:33,726 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/A in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:33,726 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/66ae5c4df92944fca22cbab39adfd4e4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/de18ba24690b483395e789038c0e68cb, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/26e17c901b9b4171be0b169241d3e08d] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=115.9 K 2024-11-26T15:25:33,727 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:33,727 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/66ae5c4df92944fca22cbab39adfd4e4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/de18ba24690b483395e789038c0e68cb, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/26e17c901b9b4171be0b169241d3e08d] 2024-11-26T15:25:33,727 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66ae5c4df92944fca22cbab39adfd4e4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732634731752 2024-11-26T15:25:33,727 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:33,728 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/B is initiating minor compaction (all files) 2024-11-26T15:25:33,728 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/B in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:33,728 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/ce27236042234386a59319b3abc6d9ef, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/6201e4fa90104fc7819b6f7c3cc3d7c4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/b6e1bf7be74746baae5f5bd9c70cd373] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=35.2 K 2024-11-26T15:25:33,729 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting de18ba24690b483395e789038c0e68cb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732634731811 2024-11-26T15:25:33,729 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting ce27236042234386a59319b3abc6d9ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732634731752 2024-11-26T15:25:33,729 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26e17c901b9b4171be0b169241d3e08d, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732634732460 2024-11-26T15:25:33,729 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 6201e4fa90104fc7819b6f7c3cc3d7c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732634731813 2024-11-26T15:25:33,730 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting b6e1bf7be74746baae5f5bd9c70cd373, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732634732460 2024-11-26T15:25:33,745 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:33,748 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#B#compaction#117 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:33,749 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/ba3c99c0840a49b7ad504958716843bb is 50, key is test_row_0/B:col10/1732634732460/Put/seqid=0 2024-11-26T15:25:33,751 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411263f0638b5f47b4fa580ef026ad3b3e8d9_997d8d2a525d3ba7cd152ef78c94cfd4 store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:33,755 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411263f0638b5f47b4fa580ef026ad3b3e8d9_997d8d2a525d3ba7cd152ef78c94cfd4, store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:33,756 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263f0638b5f47b4fa580ef026ad3b3e8d9_997d8d2a525d3ba7cd152ef78c94cfd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:33,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-26T15:25:33,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:33,767 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T15:25:33,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741961_1137 (size=12104) 2024-11-26T15:25:33,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741960_1136 (size=4469) 2024-11-26T15:25:33,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126286925bd15a5451eb34bbc05eaa069ba_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634733206/Put/seqid=0 2024-11-26T15:25:33,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741962_1138 (size=12154) 2024-11-26T15:25:33,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:33,795 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126286925bd15a5451eb34bbc05eaa069ba_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126286925bd15a5451eb34bbc05eaa069ba_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:33,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/3960bc94348d447b9e1c0007d5a87751, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:33,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/3960bc94348d447b9e1c0007d5a87751 is 175, key is test_row_0/A:col10/1732634733206/Put/seqid=0 2024-11-26T15:25:33,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741963_1139 (size=30955) 2024-11-26T15:25:33,807 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/3960bc94348d447b9e1c0007d5a87751 2024-11-26T15:25:33,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/a9ef100e1dbc4c58a596cf42196643c1 is 50, key is test_row_0/B:col10/1732634733206/Put/seqid=0 2024-11-26T15:25:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:33,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:33,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741964_1140 (size=12001) 2024-11-26T15:25:33,849 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/a9ef100e1dbc4c58a596cf42196643c1 2024-11-26T15:25:33,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634793856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634793859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634793861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634793862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:33,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:33,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/ef143ead8e3d45a79ab25d2101e1b1c5 is 50, key is test_row_0/C:col10/1732634733206/Put/seqid=0 2024-11-26T15:25:33,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741965_1141 (size=12001) 2024-11-26T15:25:33,878 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/ef143ead8e3d45a79ab25d2101e1b1c5 2024-11-26T15:25:33,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/3960bc94348d447b9e1c0007d5a87751 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/3960bc94348d447b9e1c0007d5a87751 2024-11-26T15:25:33,890 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/3960bc94348d447b9e1c0007d5a87751, entries=150, sequenceid=78, filesize=30.2 K 2024-11-26T15:25:33,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/a9ef100e1dbc4c58a596cf42196643c1 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/a9ef100e1dbc4c58a596cf42196643c1 2024-11-26T15:25:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T15:25:33,898 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/a9ef100e1dbc4c58a596cf42196643c1, entries=150, sequenceid=78, filesize=11.7 K 2024-11-26T15:25:33,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/ef143ead8e3d45a79ab25d2101e1b1c5 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/ef143ead8e3d45a79ab25d2101e1b1c5 2024-11-26T15:25:33,906 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/ef143ead8e3d45a79ab25d2101e1b1c5, entries=150, sequenceid=78, filesize=11.7 K 2024-11-26T15:25:33,908 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 140ms, sequenceid=78, compaction requested=true 2024-11-26T15:25:33,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:33,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:33,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-26T15:25:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-26T15:25:33,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-26T15:25:33,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1630 sec 2024-11-26T15:25:33,914 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.1690 sec 2024-11-26T15:25:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:33,976 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:33,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-26T15:25:33,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c092ef95f3774e09a0c2513abe0e7ee4_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634733976/Put/seqid=0 2024-11-26T15:25:34,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741966_1142 (size=17034) 2024-11-26T15:25:34,014 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:34,023 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c092ef95f3774e09a0c2513abe0e7ee4_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c092ef95f3774e09a0c2513abe0e7ee4_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:34,024 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/a8034cc6fc6f474fa3c7f25714046593, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:34,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/a8034cc6fc6f474fa3c7f25714046593 is 175, key is test_row_0/A:col10/1732634733976/Put/seqid=0 2024-11-26T15:25:34,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634794021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634794022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634794024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634794027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741967_1143 (size=48139) 2024-11-26T15:25:34,037 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/a8034cc6fc6f474fa3c7f25714046593 2024-11-26T15:25:34,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634794032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,042 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in 997d8d2a525d3ba7cd152ef78c94cfd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:25:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/e2109e5cf6ef46e9b6f27f46a859b914 is 50, key is test_row_0/B:col10/1732634733976/Put/seqid=0 2024-11-26T15:25:34,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741968_1144 (size=12001) 2024-11-26T15:25:34,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/e2109e5cf6ef46e9b6f27f46a859b914 2024-11-26T15:25:34,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/9947436b861b4109a598e901f55f80ee is 50, key is test_row_0/C:col10/1732634733976/Put/seqid=0 2024-11-26T15:25:34,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741969_1145 (size=12001) 2024-11-26T15:25:34,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634794133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634794133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634794133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634794143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,170 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#A#compaction#118 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:34,171 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/087fd1afacc5472cbec25a2fe7797aa7 is 175, key is test_row_0/A:col10/1732634732460/Put/seqid=0 2024-11-26T15:25:34,181 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/ba3c99c0840a49b7ad504958716843bb as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/ba3c99c0840a49b7ad504958716843bb 2024-11-26T15:25:34,190 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/B of 997d8d2a525d3ba7cd152ef78c94cfd4 into ba3c99c0840a49b7ad504958716843bb(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:34,190 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:34,190 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/B, priority=13, startTime=1732634733725; duration=0sec 2024-11-26T15:25:34,190 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:34,190 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:B 2024-11-26T15:25:34,191 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:25:34,193 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:25:34,193 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/C is initiating minor compaction (all files) 2024-11-26T15:25:34,193 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/C in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:34,194 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/60b68554821e4cc198852cf5a5e4e295, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/8caaf450f3c54d6591b0e56d1c166235, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/72efeee9270347f1b94f898660157f38, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/ef143ead8e3d45a79ab25d2101e1b1c5] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=46.9 K 2024-11-26T15:25:34,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741970_1146 (size=31058) 2024-11-26T15:25:34,195 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 60b68554821e4cc198852cf5a5e4e295, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732634731752 2024-11-26T15:25:34,196 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 8caaf450f3c54d6591b0e56d1c166235, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732634731813 2024-11-26T15:25:34,196 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 72efeee9270347f1b94f898660157f38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732634732460 2024-11-26T15:25:34,197 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting ef143ead8e3d45a79ab25d2101e1b1c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732634733193 2024-11-26T15:25:34,203 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/087fd1afacc5472cbec25a2fe7797aa7 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/087fd1afacc5472cbec25a2fe7797aa7 2024-11-26T15:25:34,210 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#C#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:34,212 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/A of 997d8d2a525d3ba7cd152ef78c94cfd4 into 087fd1afacc5472cbec25a2fe7797aa7(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:34,212 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:34,212 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/A, priority=13, startTime=1732634733725; duration=0sec 2024-11-26T15:25:34,213 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:34,213 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:A 2024-11-26T15:25:34,216 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/fce41aa7e14b47e89d517be201b0dea8 is 50, key is test_row_0/C:col10/1732634733206/Put/seqid=0 2024-11-26T15:25:34,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741971_1147 (size=12139) 2024-11-26T15:25:34,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634794343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634794343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634794343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634794353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/9947436b861b4109a598e901f55f80ee 2024-11-26T15:25:34,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/a8034cc6fc6f474fa3c7f25714046593 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/a8034cc6fc6f474fa3c7f25714046593 2024-11-26T15:25:34,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/a8034cc6fc6f474fa3c7f25714046593, entries=250, sequenceid=92, filesize=47.0 K 2024-11-26T15:25:34,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/e2109e5cf6ef46e9b6f27f46a859b914 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/e2109e5cf6ef46e9b6f27f46a859b914 2024-11-26T15:25:34,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/e2109e5cf6ef46e9b6f27f46a859b914, entries=150, sequenceid=92, filesize=11.7 K 2024-11-26T15:25:34,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/9947436b861b4109a598e901f55f80ee as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/9947436b861b4109a598e901f55f80ee 2024-11-26T15:25:34,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/9947436b861b4109a598e901f55f80ee, entries=150, sequenceid=92, filesize=11.7 K 2024-11-26T15:25:34,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 595ms, sequenceid=92, compaction requested=true 2024-11-26T15:25:34,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:34,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:A, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:34,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:34,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:B, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:34,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T15:25:34,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:34,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T15:25:34,572 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-26T15:25:34,572 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T15:25:34,572 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T15:25:34,573 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. because compaction request was cancelled 2024-11-26T15:25:34,573 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:C 2024-11-26T15:25:34,573 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:34,574 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:34,575 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/A is initiating minor compaction (all files) 2024-11-26T15:25:34,575 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/A in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:34,575 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/087fd1afacc5472cbec25a2fe7797aa7, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/3960bc94348d447b9e1c0007d5a87751, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/a8034cc6fc6f474fa3c7f25714046593] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=107.6 K 2024-11-26T15:25:34,575 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:34,575 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/087fd1afacc5472cbec25a2fe7797aa7, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/3960bc94348d447b9e1c0007d5a87751, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/a8034cc6fc6f474fa3c7f25714046593] 2024-11-26T15:25:34,575 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 087fd1afacc5472cbec25a2fe7797aa7, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732634732460 2024-11-26T15:25:34,576 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3960bc94348d447b9e1c0007d5a87751, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732634733193 2024-11-26T15:25:34,578 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8034cc6fc6f474fa3c7f25714046593, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732634733855 2024-11-26T15:25:34,588 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:34,590 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126c9410c12a6334715a099e16d3f1361e7_997d8d2a525d3ba7cd152ef78c94cfd4 store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:34,596 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126c9410c12a6334715a099e16d3f1361e7_997d8d2a525d3ba7cd152ef78c94cfd4, store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:34,597 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c9410c12a6334715a099e16d3f1361e7_997d8d2a525d3ba7cd152ef78c94cfd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:34,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741972_1148 (size=4469) 2024-11-26T15:25:34,613 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#A#compaction#126 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:34,614 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/4e109fda2c694cb9a9786aaf54db5f89 is 175, key is test_row_0/A:col10/1732634733976/Put/seqid=0 2024-11-26T15:25:34,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741973_1149 (size=31161) 2024-11-26T15:25:34,645 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/fce41aa7e14b47e89d517be201b0dea8 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/fce41aa7e14b47e89d517be201b0dea8 2024-11-26T15:25:34,649 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/4e109fda2c694cb9a9786aaf54db5f89 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/4e109fda2c694cb9a9786aaf54db5f89 2024-11-26T15:25:34,656 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/C of 997d8d2a525d3ba7cd152ef78c94cfd4 into fce41aa7e14b47e89d517be201b0dea8(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:34,656 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:34,656 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/C, priority=12, startTime=1732634733726; duration=0sec 2024-11-26T15:25:34,656 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:34,656 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:C 2024-11-26T15:25:34,656 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:34,658 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:34,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T15:25:34,659 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:34,659 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/A of 997d8d2a525d3ba7cd152ef78c94cfd4 into 4e109fda2c694cb9a9786aaf54db5f89(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:34,659 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/B is initiating minor compaction (all files) 2024-11-26T15:25:34,659 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:34,659 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/A, priority=13, startTime=1732634734571; duration=0sec 2024-11-26T15:25:34,659 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/B in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:34,659 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:34,659 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/ba3c99c0840a49b7ad504958716843bb, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/a9ef100e1dbc4c58a596cf42196643c1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/e2109e5cf6ef46e9b6f27f46a859b914] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=35.3 K 2024-11-26T15:25:34,659 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:A 2024-11-26T15:25:34,660 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting ba3c99c0840a49b7ad504958716843bb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732634732460 2024-11-26T15:25:34,662 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting a9ef100e1dbc4c58a596cf42196643c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732634733193 2024-11-26T15:25:34,663 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting e2109e5cf6ef46e9b6f27f46a859b914, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732634733855 2024-11-26T15:25:34,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265a98b5000c334e8caf9e142cee837a08_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634734022/Put/seqid=0 2024-11-26T15:25:34,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634794674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634794676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634794677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,686 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#B#compaction#128 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:34,686 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/43023c337d0f44d6a4f17c15f41b8966 is 50, key is test_row_0/B:col10/1732634733976/Put/seqid=0 2024-11-26T15:25:34,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634794682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741974_1150 (size=17034) 2024-11-26T15:25:34,703 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:34,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741975_1151 (size=12207) 2024-11-26T15:25:34,708 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265a98b5000c334e8caf9e142cee837a08_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265a98b5000c334e8caf9e142cee837a08_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:34,710 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/e3b820b8174b44438f69defacd138ce6, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:34,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/e3b820b8174b44438f69defacd138ce6 is 175, key is test_row_0/A:col10/1732634734022/Put/seqid=0 2024-11-26T15:25:34,717 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/43023c337d0f44d6a4f17c15f41b8966 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/43023c337d0f44d6a4f17c15f41b8966 2024-11-26T15:25:34,727 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/B of 997d8d2a525d3ba7cd152ef78c94cfd4 into 43023c337d0f44d6a4f17c15f41b8966(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:34,727 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:34,727 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/B, priority=13, startTime=1732634734571; duration=0sec 2024-11-26T15:25:34,727 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:34,727 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:B 2024-11-26T15:25:34,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741976_1152 (size=48139) 2024-11-26T15:25:34,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634794793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634794793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634794793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634794793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:35,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:35,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634795002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:35,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634795003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:35,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:35,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634795003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:35,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:35,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634795002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:35,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:35,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:35,141 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/e3b820b8174b44438f69defacd138ce6 2024-11-26T15:25:35,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/c83df15cf4fe45a6952b9b19fcea1abf is 50, key is test_row_0/B:col10/1732634734022/Put/seqid=0 2024-11-26T15:25:35,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741977_1153 (size=12001) 2024-11-26T15:25:35,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634795312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634795312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634795312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634795312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:35,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/c83df15cf4fe45a6952b9b19fcea1abf 2024-11-26T15:25:35,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/8626f6982ef9487295f6d0cbb2824183 is 50, key is test_row_0/C:col10/1732634734022/Put/seqid=0 2024-11-26T15:25:35,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741978_1154 (size=12001) 2024-11-26T15:25:35,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/8626f6982ef9487295f6d0cbb2824183 2024-11-26T15:25:35,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/e3b820b8174b44438f69defacd138ce6 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e3b820b8174b44438f69defacd138ce6 2024-11-26T15:25:35,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e3b820b8174b44438f69defacd138ce6, entries=250, sequenceid=121, filesize=47.0 K 2024-11-26T15:25:35,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/c83df15cf4fe45a6952b9b19fcea1abf as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/c83df15cf4fe45a6952b9b19fcea1abf 2024-11-26T15:25:35,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/c83df15cf4fe45a6952b9b19fcea1abf, entries=150, sequenceid=121, filesize=11.7 K 2024-11-26T15:25:35,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/8626f6982ef9487295f6d0cbb2824183 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/8626f6982ef9487295f6d0cbb2824183 2024-11-26T15:25:35,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/8626f6982ef9487295f6d0cbb2824183, entries=150, sequenceid=121, filesize=11.7 K 2024-11-26T15:25:35,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 950ms, sequenceid=121, compaction requested=true 2024-11-26T15:25:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:35,609 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T15:25:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:35,609 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T15:25:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:35,609 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T15:25:35,610 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T15:25:35,610 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T15:25:35,610 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T15:25:35,610 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. because compaction request was cancelled 2024-11-26T15:25:35,610 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. because compaction request was cancelled 2024-11-26T15:25:35,610 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:A 2024-11-26T15:25:35,610 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:B 2024-11-26T15:25:35,610 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:35,611 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:35,611 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/C is initiating minor compaction (all files) 2024-11-26T15:25:35,612 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/C in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:35,612 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/fce41aa7e14b47e89d517be201b0dea8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/9947436b861b4109a598e901f55f80ee, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/8626f6982ef9487295f6d0cbb2824183] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=35.3 K 2024-11-26T15:25:35,612 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting fce41aa7e14b47e89d517be201b0dea8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732634733193 2024-11-26T15:25:35,612 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9947436b861b4109a598e901f55f80ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732634733855 2024-11-26T15:25:35,613 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8626f6982ef9487295f6d0cbb2824183, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732634734021 2024-11-26T15:25:35,623 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#C#compaction#131 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:35,623 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/5aeeeeb72188475599213886f786f183 is 50, key is test_row_0/C:col10/1732634734022/Put/seqid=0 2024-11-26T15:25:35,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741979_1155 (size=12241) 2024-11-26T15:25:35,640 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/5aeeeeb72188475599213886f786f183 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/5aeeeeb72188475599213886f786f183 2024-11-26T15:25:35,668 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/C of 997d8d2a525d3ba7cd152ef78c94cfd4 into 5aeeeeb72188475599213886f786f183(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:35,668 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:35,668 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/C, priority=13, startTime=1732634735609; duration=0sec 2024-11-26T15:25:35,669 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:35,669 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:C 2024-11-26T15:25:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:35,824 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:35,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-26T15:25:35,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262f03fabe898744d19634fae94d7eabdf_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634735824/Put/seqid=0 2024-11-26T15:25:35,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741980_1156 (size=12304) 2024-11-26T15:25:35,854 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:35,860 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262f03fabe898744d19634fae94d7eabdf_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262f03fabe898744d19634fae94d7eabdf_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:35,861 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/14c9474da0154333a0b7849f135efe20, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:35,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/14c9474da0154333a0b7849f135efe20 is 175, key is test_row_0/A:col10/1732634735824/Put/seqid=0 2024-11-26T15:25:35,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634795859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:35,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:35,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634795866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634795864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:35,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634795867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:35,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741981_1157 (size=31105) 2024-11-26T15:25:35,879 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/14c9474da0154333a0b7849f135efe20 2024-11-26T15:25:35,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/8a391479ce8b47579917f930eb9e2745 is 50, key is test_row_0/B:col10/1732634735824/Put/seqid=0 2024-11-26T15:25:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-26T15:25:35,903 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:35,903 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:25:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-26T15:25:35,908 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:25:35,909 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:25:35,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:25:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T15:25:35,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741982_1158 (size=12151) 2024-11-26T15:25:35,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634795972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:35,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:35,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634795983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634795983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634795983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T15:25:36,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T15:25:36,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:36,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:36,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:36,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:36,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:36,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634796062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:36,067 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in 997d8d2a525d3ba7cd152ef78c94cfd4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:25:36,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:36,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634796182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:36,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:36,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634796192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:36,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:36,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:36,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634796193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634796193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:36,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:36,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:36,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T15:25:36,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:36,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:36,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:36,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:36,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:36,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T15:25:36,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T15:25:36,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:36,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:36,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:36,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:36,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:36,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/8a391479ce8b47579917f930eb9e2745 2024-11-26T15:25:36,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/b36d5a925b994866b8a04bbe8000076a is 50, key is test_row_0/C:col10/1732634735824/Put/seqid=0 2024-11-26T15:25:36,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741983_1159 (size=12151) 2024-11-26T15:25:36,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634796492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:36,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:36,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634796502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:36,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:36,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634796502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:36,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:36,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634796502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:36,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:36,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T15:25:36,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:36,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:36,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:36,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:36,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T15:25:36,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T15:25:36,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:36,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:36,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:36,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:36,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:36,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/b36d5a925b994866b8a04bbe8000076a 2024-11-26T15:25:36,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/14c9474da0154333a0b7849f135efe20 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/14c9474da0154333a0b7849f135efe20 2024-11-26T15:25:36,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/14c9474da0154333a0b7849f135efe20, entries=150, sequenceid=137, filesize=30.4 K 2024-11-26T15:25:36,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/8a391479ce8b47579917f930eb9e2745 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/8a391479ce8b47579917f930eb9e2745 2024-11-26T15:25:36,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/8a391479ce8b47579917f930eb9e2745, entries=150, sequenceid=137, filesize=11.9 K 2024-11-26T15:25:36,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/b36d5a925b994866b8a04bbe8000076a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/b36d5a925b994866b8a04bbe8000076a 2024-11-26T15:25:36,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/b36d5a925b994866b8a04bbe8000076a, entries=150, sequenceid=137, filesize=11.9 K 2024-11-26T15:25:36,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 1002ms, sequenceid=137, compaction requested=true 2024-11-26T15:25:36,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:36,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:36,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:36,826 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:36,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:36,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:36,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:36,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T15:25:36,827 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:36,829 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:36,829 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:36,829 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/A is initiating minor compaction (all files) 2024-11-26T15:25:36,829 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/B is initiating minor compaction (all files) 2024-11-26T15:25:36,829 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/A in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:36,829 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/B in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:36,829 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/43023c337d0f44d6a4f17c15f41b8966, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/c83df15cf4fe45a6952b9b19fcea1abf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/8a391479ce8b47579917f930eb9e2745] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=35.5 K 2024-11-26T15:25:36,829 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/4e109fda2c694cb9a9786aaf54db5f89, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e3b820b8174b44438f69defacd138ce6, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/14c9474da0154333a0b7849f135efe20] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=107.8 K 2024-11-26T15:25:36,829 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:36,829 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/4e109fda2c694cb9a9786aaf54db5f89, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e3b820b8174b44438f69defacd138ce6, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/14c9474da0154333a0b7849f135efe20] 2024-11-26T15:25:36,830 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 43023c337d0f44d6a4f17c15f41b8966, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732634733855 2024-11-26T15:25:36,830 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e109fda2c694cb9a9786aaf54db5f89, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732634733855 2024-11-26T15:25:36,831 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3b820b8174b44438f69defacd138ce6, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732634734017 2024-11-26T15:25:36,831 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting c83df15cf4fe45a6952b9b19fcea1abf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732634734021 2024-11-26T15:25:36,831 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14c9474da0154333a0b7849f135efe20, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732634735823 2024-11-26T15:25:36,831 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a391479ce8b47579917f930eb9e2745, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732634735823 2024-11-26T15:25:36,841 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:36,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T15:25:36,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:36,846 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-26T15:25:36,857 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112633c5d08624264da886380075d8f3fc24_997d8d2a525d3ba7cd152ef78c94cfd4 store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:36,858 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#B#compaction#136 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:36,859 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/f5daf56ed73c428e864a9d60951dcd98 is 50, key is test_row_0/B:col10/1732634735824/Put/seqid=0 2024-11-26T15:25:36,859 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112633c5d08624264da886380075d8f3fc24_997d8d2a525d3ba7cd152ef78c94cfd4, store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:36,859 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112633c5d08624264da886380075d8f3fc24_997d8d2a525d3ba7cd152ef78c94cfd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:36,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e8d7327a7f27485d9e86e7c5df4d55cd_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634735865/Put/seqid=0 2024-11-26T15:25:36,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741984_1160 (size=12459) 2024-11-26T15:25:36,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741985_1161 (size=4469) 2024-11-26T15:25:36,883 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#A#compaction#135 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:36,883 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/7736efe8ecb447059e480b71f1c19dd2 is 175, key is test_row_0/A:col10/1732634735824/Put/seqid=0 2024-11-26T15:25:36,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741986_1162 (size=12304) 2024-11-26T15:25:36,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741987_1163 (size=31413) 2024-11-26T15:25:36,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:36,914 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e8d7327a7f27485d9e86e7c5df4d55cd_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e8d7327a7f27485d9e86e7c5df4d55cd_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:36,919 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/7736efe8ecb447059e480b71f1c19dd2 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7736efe8ecb447059e480b71f1c19dd2 2024-11-26T15:25:36,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/e0170a8d52704a5d9e02ed9cb1e495c0, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:36,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/e0170a8d52704a5d9e02ed9cb1e495c0 is 175, key is test_row_0/A:col10/1732634735865/Put/seqid=0 2024-11-26T15:25:36,930 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/A of 997d8d2a525d3ba7cd152ef78c94cfd4 into 7736efe8ecb447059e480b71f1c19dd2(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:36,930 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:36,930 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/A, priority=13, startTime=1732634736826; duration=0sec 2024-11-26T15:25:36,930 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:36,930 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:A 2024-11-26T15:25:36,930 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T15:25:36,931 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T15:25:36,931 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T15:25:36,931 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. because compaction request was cancelled 2024-11-26T15:25:36,931 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:C 2024-11-26T15:25:36,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741988_1164 (size=31105) 2024-11-26T15:25:37,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:37,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:37,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:37,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634797031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:37,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:37,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634797031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:37,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:37,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:37,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634797033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:37,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:37,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:37,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634797038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T15:25:37,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634797142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:37,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634797143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634797143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:37,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634797152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:37,280 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/f5daf56ed73c428e864a9d60951dcd98 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/f5daf56ed73c428e864a9d60951dcd98 2024-11-26T15:25:37,288 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/B of 997d8d2a525d3ba7cd152ef78c94cfd4 into f5daf56ed73c428e864a9d60951dcd98(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:37,288 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:37,288 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/B, priority=13, startTime=1732634736826; duration=0sec 2024-11-26T15:25:37,288 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:37,288 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:B 2024-11-26T15:25:37,347 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/e0170a8d52704a5d9e02ed9cb1e495c0 2024-11-26T15:25:37,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634797353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:37,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634797353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:37,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634797353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:37,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:37,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:37,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/9c6c7f4530ea44d59cb521b06d4ef267 is 50, key is test_row_0/B:col10/1732634735865/Put/seqid=0 2024-11-26T15:25:37,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741989_1165 (size=12151) 2024-11-26T15:25:37,359 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/9c6c7f4530ea44d59cb521b06d4ef267 2024-11-26T15:25:37,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634797362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:37,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/72c317b00c4e45e393348098426da0c6 is 50, key is test_row_0/C:col10/1732634735865/Put/seqid=0 2024-11-26T15:25:37,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741990_1166 (size=12151) 2024-11-26T15:25:37,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634797662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634797663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634797663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:37,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634797672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:37,778 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/72c317b00c4e45e393348098426da0c6 2024-11-26T15:25:37,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/e0170a8d52704a5d9e02ed9cb1e495c0 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e0170a8d52704a5d9e02ed9cb1e495c0 2024-11-26T15:25:37,793 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e0170a8d52704a5d9e02ed9cb1e495c0, entries=150, sequenceid=159, filesize=30.4 K 2024-11-26T15:25:37,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/9c6c7f4530ea44d59cb521b06d4ef267 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/9c6c7f4530ea44d59cb521b06d4ef267 2024-11-26T15:25:37,800 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/9c6c7f4530ea44d59cb521b06d4ef267, entries=150, sequenceid=159, filesize=11.9 K 2024-11-26T15:25:37,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/72c317b00c4e45e393348098426da0c6 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/72c317b00c4e45e393348098426da0c6 2024-11-26T15:25:37,811 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/72c317b00c4e45e393348098426da0c6, entries=150, sequenceid=159, filesize=11.9 K 2024-11-26T15:25:37,814 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 968ms, sequenceid=159, compaction requested=true 2024-11-26T15:25:37,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:37,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:37,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-26T15:25:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-26T15:25:37,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-26T15:25:37,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9070 sec 2024-11-26T15:25:37,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.9160 sec 2024-11-26T15:25:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T15:25:38,062 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:38,063 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:25:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-26T15:25:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T15:25:38,064 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:25:38,065 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:25:38,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:25:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T15:25:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:38,175 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:38,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-26T15:25:38,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d930a408f3f246fba4d487e0eb25f233_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634738175/Put/seqid=0 2024-11-26T15:25:38,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741991_1167 (size=17284) 2024-11-26T15:25:38,204 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:38,210 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d930a408f3f246fba4d487e0eb25f233_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d930a408f3f246fba4d487e0eb25f233_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:38,211 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/8892579a190f4dc4a1972c99f8296bfe, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:38,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/8892579a190f4dc4a1972c99f8296bfe is 175, key is test_row_0/A:col10/1732634738175/Put/seqid=0 2024-11-26T15:25:38,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T15:25:38,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:38,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:38,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:38,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634798212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:38,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634798213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:38,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634798214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:38,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634798223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:38,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:38,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741992_1168 (size=48389) 2024-11-26T15:25:38,243 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/8892579a190f4dc4a1972c99f8296bfe 2024-11-26T15:25:38,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/df10ad4b6b224daab3d5a2382ec33cf1 is 50, key is test_row_0/B:col10/1732634738175/Put/seqid=0 2024-11-26T15:25:38,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741993_1169 (size=12151) 2024-11-26T15:25:38,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634798332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634798332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:38,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634798333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:38,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634798333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:38,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T15:25:38,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:38,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:38,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:38,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T15:25:38,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T15:25:38,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:38,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:38,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:38,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634798543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:38,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634798543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634798543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:38,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634798543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:38,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/df10ad4b6b224daab3d5a2382ec33cf1 2024-11-26T15:25:38,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/27992ebc6ab944ea81fe57497eeb51df is 50, key is test_row_0/C:col10/1732634738175/Put/seqid=0 2024-11-26T15:25:38,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741994_1170 (size=12151) 2024-11-26T15:25:38,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T15:25:38,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:38,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:38,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:38,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T15:25:38,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T15:25:38,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:38,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:38,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:38,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634798853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:38,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:38,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634798853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:38,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:38,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:38,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634798862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:38,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:38,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634798862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:38,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:38,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:38,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:38,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T15:25:38,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:38,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:38,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:38,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:38,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:39,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/27992ebc6ab944ea81fe57497eeb51df 2024-11-26T15:25:39,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/8892579a190f4dc4a1972c99f8296bfe as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/8892579a190f4dc4a1972c99f8296bfe 2024-11-26T15:25:39,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/8892579a190f4dc4a1972c99f8296bfe, entries=250, sequenceid=178, filesize=47.3 K 2024-11-26T15:25:39,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/df10ad4b6b224daab3d5a2382ec33cf1 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/df10ad4b6b224daab3d5a2382ec33cf1 2024-11-26T15:25:39,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/df10ad4b6b224daab3d5a2382ec33cf1, entries=150, sequenceid=178, filesize=11.9 K 2024-11-26T15:25:39,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/27992ebc6ab944ea81fe57497eeb51df as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/27992ebc6ab944ea81fe57497eeb51df 2024-11-26T15:25:39,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/27992ebc6ab944ea81fe57497eeb51df, entries=150, sequenceid=178, filesize=11.9 K 2024-11-26T15:25:39,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 927ms, sequenceid=178, compaction requested=true 2024-11-26T15:25:39,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:39,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:39,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:39,103 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:39,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:39,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:39,103 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:39,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:39,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:39,104 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:39,104 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/A is initiating minor compaction (all files) 2024-11-26T15:25:39,104 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:39,104 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/A in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:39,104 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/B is initiating minor compaction (all files) 2024-11-26T15:25:39,104 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7736efe8ecb447059e480b71f1c19dd2, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e0170a8d52704a5d9e02ed9cb1e495c0, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/8892579a190f4dc4a1972c99f8296bfe] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=108.3 K 2024-11-26T15:25:39,104 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/B in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:39,104 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:39,104 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/f5daf56ed73c428e864a9d60951dcd98, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/9c6c7f4530ea44d59cb521b06d4ef267, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/df10ad4b6b224daab3d5a2382ec33cf1] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=35.9 K 2024-11-26T15:25:39,104 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7736efe8ecb447059e480b71f1c19dd2, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e0170a8d52704a5d9e02ed9cb1e495c0, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/8892579a190f4dc4a1972c99f8296bfe] 2024-11-26T15:25:39,104 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting f5daf56ed73c428e864a9d60951dcd98, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732634735823 2024-11-26T15:25:39,104 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7736efe8ecb447059e480b71f1c19dd2, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732634735823 2024-11-26T15:25:39,105 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0170a8d52704a5d9e02ed9cb1e495c0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732634735857 2024-11-26T15:25:39,105 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c6c7f4530ea44d59cb521b06d4ef267, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732634735857 2024-11-26T15:25:39,105 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8892579a190f4dc4a1972c99f8296bfe, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732634737031 2024-11-26T15:25:39,105 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting df10ad4b6b224daab3d5a2382ec33cf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732634737031 2024-11-26T15:25:39,109 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:39,111 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#B#compaction#143 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:39,112 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/cc6b24fd9f5d4e199212d2c5fe15a7c9 is 50, key is test_row_0/B:col10/1732634738175/Put/seqid=0 2024-11-26T15:25:39,124 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126071d82c68e9049ef94cb921de97d3789_997d8d2a525d3ba7cd152ef78c94cfd4 store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:39,126 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126071d82c68e9049ef94cb921de97d3789_997d8d2a525d3ba7cd152ef78c94cfd4, store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:39,127 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126071d82c68e9049ef94cb921de97d3789_997d8d2a525d3ba7cd152ef78c94cfd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:39,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T15:25:39,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:39,137 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-26T15:25:39,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741995_1171 (size=12561) 2024-11-26T15:25:39,149 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/cc6b24fd9f5d4e199212d2c5fe15a7c9 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/cc6b24fd9f5d4e199212d2c5fe15a7c9 2024-11-26T15:25:39,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741996_1172 (size=4469) 2024-11-26T15:25:39,165 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/B of 997d8d2a525d3ba7cd152ef78c94cfd4 into cc6b24fd9f5d4e199212d2c5fe15a7c9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:39,165 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:39,165 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/B, priority=13, startTime=1732634739103; duration=0sec 2024-11-26T15:25:39,165 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:39,165 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:B 2024-11-26T15:25:39,165 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:25:39,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a8b2cbaede974f07a81cad6d877eaee2_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634738210/Put/seqid=0 2024-11-26T15:25:39,169 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#A#compaction#144 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:39,169 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/7cc6dd53676a47a397ad08061523258f is 175, key is test_row_0/A:col10/1732634738175/Put/seqid=0 2024-11-26T15:25:39,171 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:25:39,171 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/C is initiating minor compaction (all files) 2024-11-26T15:25:39,171 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/C in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:39,172 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/5aeeeeb72188475599213886f786f183, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/b36d5a925b994866b8a04bbe8000076a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/72c317b00c4e45e393348098426da0c6, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/27992ebc6ab944ea81fe57497eeb51df] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=47.6 K 2024-11-26T15:25:39,172 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aeeeeb72188475599213886f786f183, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732634734021 2024-11-26T15:25:39,173 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting b36d5a925b994866b8a04bbe8000076a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732634735823 2024-11-26T15:25:39,173 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 72c317b00c4e45e393348098426da0c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732634735857 2024-11-26T15:25:39,174 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 27992ebc6ab944ea81fe57497eeb51df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732634737031 2024-11-26T15:25:39,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741997_1173 (size=12304) 2024-11-26T15:25:39,192 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#C#compaction#146 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:39,196 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/fb6eaecee31342919361dad29bf38a21 is 50, key is test_row_0/C:col10/1732634738175/Put/seqid=0 2024-11-26T15:25:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T15:25:39,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741998_1174 (size=31515) 2024-11-26T15:25:39,213 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/7cc6dd53676a47a397ad08061523258f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7cc6dd53676a47a397ad08061523258f 2024-11-26T15:25:39,221 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/A of 997d8d2a525d3ba7cd152ef78c94cfd4 into 7cc6dd53676a47a397ad08061523258f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:39,221 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:39,221 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/A, priority=13, startTime=1732634739102; duration=0sec 2024-11-26T15:25:39,222 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:39,222 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:A 2024-11-26T15:25:39,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741999_1175 (size=12527) 2024-11-26T15:25:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:39,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:39,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634799398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:39,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634799398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634799399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:39,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634799399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:39,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634799513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634799513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634799513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:39,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634799513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:39,596 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a8b2cbaede974f07a81cad6d877eaee2_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a8b2cbaede974f07a81cad6d877eaee2_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:39,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/fb9b020ef6924697ba014a153948d83f, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:39,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/fb9b020ef6924697ba014a153948d83f is 175, key is test_row_0/A:col10/1732634738210/Put/seqid=0 2024-11-26T15:25:39,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742000_1176 (size=31105) 2024-11-26T15:25:39,603 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/fb9b020ef6924697ba014a153948d83f 2024-11-26T15:25:39,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/5904a4d5697049eca107617adb042ddf is 50, key is test_row_0/B:col10/1732634738210/Put/seqid=0 2024-11-26T15:25:39,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742001_1177 (size=12151) 2024-11-26T15:25:39,619 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/5904a4d5697049eca107617adb042ddf 2024-11-26T15:25:39,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/b8716d63e2d34c54b15347bb35edc1db is 50, key is test_row_0/C:col10/1732634738210/Put/seqid=0 2024-11-26T15:25:39,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742002_1178 (size=12151) 2024-11-26T15:25:39,641 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/fb6eaecee31342919361dad29bf38a21 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/fb6eaecee31342919361dad29bf38a21 2024-11-26T15:25:39,646 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/C of 997d8d2a525d3ba7cd152ef78c94cfd4 into fb6eaecee31342919361dad29bf38a21(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:39,646 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:39,646 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/C, priority=12, startTime=1732634739103; duration=0sec 2024-11-26T15:25:39,646 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:39,646 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:C 2024-11-26T15:25:39,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634799723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:39,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634799723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:39,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:39,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:39,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634799723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:39,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:39,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:39,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634799723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:39,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:39,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:40,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634800032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:40,035 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/b8716d63e2d34c54b15347bb35edc1db 2024-11-26T15:25:40,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:40,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634800033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:40,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634800033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634800033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/fb9b020ef6924697ba014a153948d83f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/fb9b020ef6924697ba014a153948d83f 2024-11-26T15:25:40,045 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/fb9b020ef6924697ba014a153948d83f, entries=150, sequenceid=197, filesize=30.4 K 2024-11-26T15:25:40,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/5904a4d5697049eca107617adb042ddf as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/5904a4d5697049eca107617adb042ddf 2024-11-26T15:25:40,050 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/5904a4d5697049eca107617adb042ddf, entries=150, sequenceid=197, filesize=11.9 K 2024-11-26T15:25:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/b8716d63e2d34c54b15347bb35edc1db as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/b8716d63e2d34c54b15347bb35edc1db 2024-11-26T15:25:40,056 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/b8716d63e2d34c54b15347bb35edc1db, entries=150, sequenceid=197, filesize=11.9 K 2024-11-26T15:25:40,057 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 920ms, sequenceid=197, compaction requested=false 2024-11-26T15:25:40,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:40,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:40,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-26T15:25:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-26T15:25:40,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-26T15:25:40,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9930 sec 2024-11-26T15:25:40,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.9980 sec 2024-11-26T15:25:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:40,115 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:40,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-26T15:25:40,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126395f202ebb3e4cf487bee2251e5dd6a9_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634739397/Put/seqid=0 2024-11-26T15:25:40,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742003_1179 (size=14794) 2024-11-26T15:25:40,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634800181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:40,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T15:25:40,213 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:40,213 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:25:40,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-26T15:25:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T15:25:40,215 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:25:40,216 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:25:40,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:25:40,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634800292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:40,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T15:25:40,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T15:25:40,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:40,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:40,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:40,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:40,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:40,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634800502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:40,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:40,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T15:25:40,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:40,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:40,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:40,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:40,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:40,527 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:40,531 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126395f202ebb3e4cf487bee2251e5dd6a9_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126395f202ebb3e4cf487bee2251e5dd6a9_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:40,532 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/f7539088c9664e45a7cadc19518156fb, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T15:25:40,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/f7539088c9664e45a7cadc19518156fb is 175, key is test_row_0/A:col10/1732634739397/Put/seqid=0 2024-11-26T15:25:40,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742004_1180 (size=39749) 2024-11-26T15:25:40,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634800543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:40,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:40,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634800553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:40,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:40,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634800553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:40,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634800553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:40,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T15:25:40,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:40,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:40,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:40,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:40,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:40,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634800812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:40,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T15:25:40,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:40,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:40,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:40,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:40,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T15:25:40,937 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/f7539088c9664e45a7cadc19518156fb 2024-11-26T15:25:40,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/97a717a1aa7341ca98fe3bb4ae719982 is 50, key is test_row_0/B:col10/1732634739397/Put/seqid=0 2024-11-26T15:25:40,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742005_1181 (size=12151) 2024-11-26T15:25:40,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T15:25:40,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:40,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:40,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:40,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:40,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:41,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T15:25:41,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:41,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:41,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:41,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:41,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:41,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:41,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T15:25:41,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:41,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:41,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:41,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:41,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:41,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634801322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:41,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:41,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T15:25:41,387 INFO [master/6cc2bc8bf733:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-26T15:25:41,387 INFO [master/6cc2bc8bf733:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-26T15:25:41,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/97a717a1aa7341ca98fe3bb4ae719982 2024-11-26T15:25:41,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/bd0e337b73bd4f508ae5817e635a5276 is 50, key is test_row_0/C:col10/1732634739397/Put/seqid=0 2024-11-26T15:25:41,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742006_1182 (size=12151) 2024-11-26T15:25:41,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T15:25:41,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:41,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:41,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:41,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:41,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:41,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634801552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:41,553 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in 997d8d2a525d3ba7cd152ef78c94cfd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:25:41,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:41,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:41,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:41,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634801562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:41,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634801562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:41,563 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in 997d8d2a525d3ba7cd152ef78c94cfd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:25:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:41,564 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in 997d8d2a525d3ba7cd152ef78c94cfd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:25:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:41,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634801572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:41,573 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in 997d8d2a525d3ba7cd152ef78c94cfd4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:25:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:41,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:41,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T15:25:41,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:41,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:41,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:41,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:41,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:41,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T15:25:41,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:41,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:41,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:41,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:41,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:41,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/bd0e337b73bd4f508ae5817e635a5276 2024-11-26T15:25:41,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/f7539088c9664e45a7cadc19518156fb as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/f7539088c9664e45a7cadc19518156fb 2024-11-26T15:25:41,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/f7539088c9664e45a7cadc19518156fb, entries=200, sequenceid=218, filesize=38.8 K 2024-11-26T15:25:41,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/97a717a1aa7341ca98fe3bb4ae719982 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/97a717a1aa7341ca98fe3bb4ae719982 2024-11-26T15:25:41,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/97a717a1aa7341ca98fe3bb4ae719982, entries=150, sequenceid=218, filesize=11.9 K 2024-11-26T15:25:41,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/bd0e337b73bd4f508ae5817e635a5276 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/bd0e337b73bd4f508ae5817e635a5276 2024-11-26T15:25:41,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/bd0e337b73bd4f508ae5817e635a5276, entries=150, sequenceid=218, filesize=11.9 K 2024-11-26T15:25:41,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 1719ms, sequenceid=218, compaction requested=true 2024-11-26T15:25:41,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:41,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:41,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:41,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:41,834 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:41,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:41,834 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:41,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:41,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:41,835 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:41,835 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:41,835 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/A is initiating minor compaction (all files) 2024-11-26T15:25:41,835 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/B is initiating minor compaction (all files) 2024-11-26T15:25:41,835 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/B in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:41,835 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/A in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:41,835 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7cc6dd53676a47a397ad08061523258f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/fb9b020ef6924697ba014a153948d83f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/f7539088c9664e45a7cadc19518156fb] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=100.0 K 2024-11-26T15:25:41,835 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/cc6b24fd9f5d4e199212d2c5fe15a7c9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/5904a4d5697049eca107617adb042ddf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/97a717a1aa7341ca98fe3bb4ae719982] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=36.0 K 2024-11-26T15:25:41,835 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:41,835 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7cc6dd53676a47a397ad08061523258f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/fb9b020ef6924697ba014a153948d83f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/f7539088c9664e45a7cadc19518156fb] 2024-11-26T15:25:41,836 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting cc6b24fd9f5d4e199212d2c5fe15a7c9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732634737031 2024-11-26T15:25:41,836 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cc6dd53676a47a397ad08061523258f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732634737031 2024-11-26T15:25:41,836 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 5904a4d5697049eca107617adb042ddf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732634738210 2024-11-26T15:25:41,836 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb9b020ef6924697ba014a153948d83f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732634738210 2024-11-26T15:25:41,837 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 97a717a1aa7341ca98fe3bb4ae719982, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732634739369 2024-11-26T15:25:41,837 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7539088c9664e45a7cadc19518156fb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732634739369 2024-11-26T15:25:41,842 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:41,844 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#B#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:41,844 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/0968d372a4d24709a2d8704ef8750c1e is 50, key is test_row_0/B:col10/1732634739397/Put/seqid=0 2024-11-26T15:25:41,849 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126b145e15d3f564a5baafb9d175298d3e2_997d8d2a525d3ba7cd152ef78c94cfd4 store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:41,851 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126b145e15d3f564a5baafb9d175298d3e2_997d8d2a525d3ba7cd152ef78c94cfd4, store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:41,851 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b145e15d3f564a5baafb9d175298d3e2_997d8d2a525d3ba7cd152ef78c94cfd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:41,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742008_1184 (size=4469) 2024-11-26T15:25:41,867 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#A#compaction#153 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:41,868 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/26e15d9dff814b32a99fc40f4b51fd7c is 175, key is test_row_0/A:col10/1732634739397/Put/seqid=0 2024-11-26T15:25:41,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742007_1183 (size=12663) 2024-11-26T15:25:41,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742009_1185 (size=31617) 2024-11-26T15:25:41,882 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/0968d372a4d24709a2d8704ef8750c1e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/0968d372a4d24709a2d8704ef8750c1e 2024-11-26T15:25:41,896 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/B of 997d8d2a525d3ba7cd152ef78c94cfd4 into 0968d372a4d24709a2d8704ef8750c1e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:41,896 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:41,896 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/B, priority=13, startTime=1732634741834; duration=0sec 2024-11-26T15:25:41,896 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:41,896 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:B 2024-11-26T15:25:41,896 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:41,897 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:41,897 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/C is initiating minor compaction (all files) 2024-11-26T15:25:41,897 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/C in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:41,898 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/fb6eaecee31342919361dad29bf38a21, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/b8716d63e2d34c54b15347bb35edc1db, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/bd0e337b73bd4f508ae5817e635a5276] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=36.0 K 2024-11-26T15:25:41,900 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting fb6eaecee31342919361dad29bf38a21, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732634737031 2024-11-26T15:25:41,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T15:25:41,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:41,900 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-26T15:25:41,900 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting b8716d63e2d34c54b15347bb35edc1db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732634738210 2024-11-26T15:25:41,901 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting bd0e337b73bd4f508ae5817e635a5276, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732634739369 2024-11-26T15:25:41,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112653e7b6c5ee7d4707b001d11703475ee9_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634740180/Put/seqid=0 2024-11-26T15:25:41,914 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#C#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:41,914 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/95019308e1774a8d852b915ace612b2b is 50, key is test_row_0/C:col10/1732634739397/Put/seqid=0 2024-11-26T15:25:41,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742010_1186 (size=12304) 2024-11-26T15:25:41,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742011_1187 (size=12629) 2024-11-26T15:25:42,288 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/26e15d9dff814b32a99fc40f4b51fd7c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/26e15d9dff814b32a99fc40f4b51fd7c 2024-11-26T15:25:42,293 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/A of 997d8d2a525d3ba7cd152ef78c94cfd4 into 26e15d9dff814b32a99fc40f4b51fd7c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:42,293 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:42,293 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/A, priority=13, startTime=1732634741834; duration=0sec 2024-11-26T15:25:42,294 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:42,294 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:A 2024-11-26T15:25:42,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:42,325 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112653e7b6c5ee7d4707b001d11703475ee9_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112653e7b6c5ee7d4707b001d11703475ee9_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:42,326 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/95019308e1774a8d852b915ace612b2b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/95019308e1774a8d852b915ace612b2b 2024-11-26T15:25:42,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/d9b530f3856c492c8094cfbac57522be, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:42,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/d9b530f3856c492c8094cfbac57522be is 175, key is test_row_0/A:col10/1732634740180/Put/seqid=0 2024-11-26T15:25:42,339 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/C of 997d8d2a525d3ba7cd152ef78c94cfd4 into 95019308e1774a8d852b915ace612b2b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:42,339 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:42,339 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/C, priority=13, startTime=1732634741834; duration=0sec 2024-11-26T15:25:42,339 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:42,339 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:C 2024-11-26T15:25:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:42,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:42,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742012_1188 (size=31105) 2024-11-26T15:25:42,355 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/d9b530f3856c492c8094cfbac57522be 2024-11-26T15:25:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T15:25:42,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/28d6bbac95584b929db9dbf770cf8365 is 50, key is test_row_0/B:col10/1732634740180/Put/seqid=0 2024-11-26T15:25:42,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742013_1189 (size=12151) 2024-11-26T15:25:42,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634802410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:42,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:42,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:42,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:42,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634802522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:42,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:42,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:42,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:42,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634802732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:42,803 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/28d6bbac95584b929db9dbf770cf8365 2024-11-26T15:25:42,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/4a81788276cf4da2848a88bc242846be is 50, key is test_row_0/C:col10/1732634740180/Put/seqid=0 2024-11-26T15:25:42,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742014_1190 (size=12151) 2024-11-26T15:25:43,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634803042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:43,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:43,215 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/4a81788276cf4da2848a88bc242846be 2024-11-26T15:25:43,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/d9b530f3856c492c8094cfbac57522be as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/d9b530f3856c492c8094cfbac57522be 2024-11-26T15:25:43,225 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/d9b530f3856c492c8094cfbac57522be, entries=150, sequenceid=237, filesize=30.4 K 2024-11-26T15:25:43,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/28d6bbac95584b929db9dbf770cf8365 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/28d6bbac95584b929db9dbf770cf8365 2024-11-26T15:25:43,230 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/28d6bbac95584b929db9dbf770cf8365, entries=150, sequenceid=237, filesize=11.9 K 2024-11-26T15:25:43,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/4a81788276cf4da2848a88bc242846be as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/4a81788276cf4da2848a88bc242846be 2024-11-26T15:25:43,235 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/4a81788276cf4da2848a88bc242846be, entries=150, sequenceid=237, filesize=11.9 K 2024-11-26T15:25:43,237 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 1337ms, sequenceid=237, compaction requested=false 2024-11-26T15:25:43,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:43,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:43,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-26T15:25:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-26T15:25:43,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-26T15:25:43,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0220 sec 2024-11-26T15:25:43,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 3.0270 sec 2024-11-26T15:25:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:43,555 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:43,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-26T15:25:43,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126414436872270432d880453d89694bad5_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634742385/Put/seqid=0 2024-11-26T15:25:43,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742015_1191 (size=14844) 2024-11-26T15:25:43,567 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:43,571 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126414436872270432d880453d89694bad5_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126414436872270432d880453d89694bad5_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:43,572 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/7db95863f44a4393b2aaafd811fc0179, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:43,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/7db95863f44a4393b2aaafd811fc0179 is 175, key is test_row_0/A:col10/1732634742385/Put/seqid=0 2024-11-26T15:25:43,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742016_1192 (size=39799) 2024-11-26T15:25:43,578 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/7db95863f44a4393b2aaafd811fc0179 2024-11-26T15:25:43,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/31ba4758b26a4336bee3721341b2f6d5 is 50, key is test_row_0/B:col10/1732634742385/Put/seqid=0 2024-11-26T15:25:43,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742017_1193 (size=12201) 2024-11-26T15:25:43,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/31ba4758b26a4336bee3721341b2f6d5 2024-11-26T15:25:43,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:43,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634803589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634803592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:43,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634803594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:43,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:43,603 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in 997d8d2a525d3ba7cd152ef78c94cfd4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:25:43,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:43,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634803601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:43,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:43,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634803602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:43,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/f30093d02bd3414c90977d234b292193 is 50, key is test_row_0/C:col10/1732634742385/Put/seqid=0 2024-11-26T15:25:43,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742018_1194 (size=12201) 2024-11-26T15:25:43,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634803712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:43,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634803712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634803712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:43,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634803722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:43,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:43,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634803923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:43,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634803923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634803923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:43,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634803932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/f30093d02bd3414c90977d234b292193 2024-11-26T15:25:44,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/7db95863f44a4393b2aaafd811fc0179 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7db95863f44a4393b2aaafd811fc0179 2024-11-26T15:25:44,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7db95863f44a4393b2aaafd811fc0179, entries=200, sequenceid=258, filesize=38.9 K 2024-11-26T15:25:44,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/31ba4758b26a4336bee3721341b2f6d5 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/31ba4758b26a4336bee3721341b2f6d5 2024-11-26T15:25:44,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/31ba4758b26a4336bee3721341b2f6d5, entries=150, sequenceid=258, filesize=11.9 K 2024-11-26T15:25:44,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/f30093d02bd3414c90977d234b292193 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/f30093d02bd3414c90977d234b292193 2024-11-26T15:25:44,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/f30093d02bd3414c90977d234b292193, entries=150, sequenceid=258, filesize=11.9 K 2024-11-26T15:25:44,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 495ms, sequenceid=258, compaction requested=true 2024-11-26T15:25:44,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:44,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:44,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:44,051 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:44,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:44,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:44,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:44,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T15:25:44,051 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:44,052 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:44,052 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/B is initiating minor compaction (all files) 2024-11-26T15:25:44,052 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/B in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:44,052 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:44,052 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/A is initiating minor compaction (all files) 2024-11-26T15:25:44,052 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/0968d372a4d24709a2d8704ef8750c1e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/28d6bbac95584b929db9dbf770cf8365, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/31ba4758b26a4336bee3721341b2f6d5] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=36.1 K 2024-11-26T15:25:44,052 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/A in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:44,053 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/26e15d9dff814b32a99fc40f4b51fd7c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/d9b530f3856c492c8094cfbac57522be, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7db95863f44a4393b2aaafd811fc0179] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=100.1 K 2024-11-26T15:25:44,053 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:44,053 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/26e15d9dff814b32a99fc40f4b51fd7c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/d9b530f3856c492c8094cfbac57522be, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7db95863f44a4393b2aaafd811fc0179] 2024-11-26T15:25:44,053 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 26e15d9dff814b32a99fc40f4b51fd7c, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732634739369 2024-11-26T15:25:44,053 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0968d372a4d24709a2d8704ef8750c1e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732634739369 2024-11-26T15:25:44,054 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting d9b530f3856c492c8094cfbac57522be, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732634740166 2024-11-26T15:25:44,054 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28d6bbac95584b929db9dbf770cf8365, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732634740166 2024-11-26T15:25:44,054 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 7db95863f44a4393b2aaafd811fc0179, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732634742385 2024-11-26T15:25:44,054 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31ba4758b26a4336bee3721341b2f6d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732634742385 2024-11-26T15:25:44,062 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#B#compaction#161 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:44,062 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/8556ad0c3a224bfe99b0605a93028b88 is 50, key is test_row_0/B:col10/1732634742385/Put/seqid=0 2024-11-26T15:25:44,070 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:44,073 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126476f9f36abea4ea385662d618d0e4e5f_997d8d2a525d3ba7cd152ef78c94cfd4 store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:44,075 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126476f9f36abea4ea385662d618d0e4e5f_997d8d2a525d3ba7cd152ef78c94cfd4, store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:44,075 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126476f9f36abea4ea385662d618d0e4e5f_997d8d2a525d3ba7cd152ef78c94cfd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:44,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742019_1195 (size=12815) 2024-11-26T15:25:44,089 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/8556ad0c3a224bfe99b0605a93028b88 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/8556ad0c3a224bfe99b0605a93028b88 2024-11-26T15:25:44,095 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/B of 997d8d2a525d3ba7cd152ef78c94cfd4 into 8556ad0c3a224bfe99b0605a93028b88(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:44,095 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:44,095 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/B, priority=13, startTime=1732634744051; duration=0sec 2024-11-26T15:25:44,095 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:44,095 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:B 2024-11-26T15:25:44,095 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:25:44,098 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36981 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:25:44,098 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/C is initiating minor compaction (all files) 2024-11-26T15:25:44,098 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/C in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:44,098 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/95019308e1774a8d852b915ace612b2b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/4a81788276cf4da2848a88bc242846be, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/f30093d02bd3414c90977d234b292193] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=36.1 K 2024-11-26T15:25:44,099 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95019308e1774a8d852b915ace612b2b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732634739369 2024-11-26T15:25:44,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742020_1196 (size=4469) 2024-11-26T15:25:44,099 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a81788276cf4da2848a88bc242846be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732634740166 2024-11-26T15:25:44,100 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting f30093d02bd3414c90977d234b292193, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732634742385 2024-11-26T15:25:44,112 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#C#compaction#163 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:44,113 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/c9b68488c3e5447e855b8fb9a704ff9f is 50, key is test_row_0/C:col10/1732634742385/Put/seqid=0 2024-11-26T15:25:44,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742021_1197 (size=12781) 2024-11-26T15:25:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:44,234 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:44,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-26T15:25:44,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260d3a9fc2ba2e4c6f8c823ff9c8ac61f8_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634744234/Put/seqid=0 2024-11-26T15:25:44,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742022_1198 (size=12454) 2024-11-26T15:25:44,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634804253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:44,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:44,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634804254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:44,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:44,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634804256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634804254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:44,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634804362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T15:25:44,373 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:44,373 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:25:44,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634804373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:44,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634804373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:44,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-26T15:25:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T15:25:44,375 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:25:44,375 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:25:44,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:25:44,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634804373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:44,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T15:25:44,500 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#A#compaction#162 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:44,501 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/1296fb951ea24db5a8df80fa68df6082 is 175, key is test_row_0/A:col10/1732634742385/Put/seqid=0 2024-11-26T15:25:44,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742023_1199 (size=31769) 2024-11-26T15:25:44,509 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/1296fb951ea24db5a8df80fa68df6082 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/1296fb951ea24db5a8df80fa68df6082 2024-11-26T15:25:44,514 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/A of 997d8d2a525d3ba7cd152ef78c94cfd4 into 1296fb951ea24db5a8df80fa68df6082(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:44,514 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:44,514 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/A, priority=13, startTime=1732634744050; duration=0sec 2024-11-26T15:25:44,515 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:44,515 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:A 2024-11-26T15:25:44,525 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/c9b68488c3e5447e855b8fb9a704ff9f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/c9b68488c3e5447e855b8fb9a704ff9f 2024-11-26T15:25:44,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T15:25:44,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:44,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:44,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:44,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:44,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:44,531 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/C of 997d8d2a525d3ba7cd152ef78c94cfd4 into c9b68488c3e5447e855b8fb9a704ff9f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:44,531 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:44,531 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/C, priority=13, startTime=1732634744051; duration=0sec 2024-11-26T15:25:44,531 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:44,531 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:C 2024-11-26T15:25:44,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634804572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634804583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:44,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634804583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634804583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,654 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:44,658 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260d3a9fc2ba2e4c6f8c823ff9c8ac61f8_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260d3a9fc2ba2e4c6f8c823ff9c8ac61f8_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:44,659 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/a013d2208a1c40f5b5e7e91b0059589b, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:44,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/a013d2208a1c40f5b5e7e91b0059589b is 175, key is test_row_0/A:col10/1732634744234/Put/seqid=0 2024-11-26T15:25:44,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742024_1200 (size=31255) 2024-11-26T15:25:44,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T15:25:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:44,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:44,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T15:25:44,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T15:25:44,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:44,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:44,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:44,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:44,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:44,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:44,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:44,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634804882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:44,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:44,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634804903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634804903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634804912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:44,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T15:25:44,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:44,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:44,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:44,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:44,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:44,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T15:25:45,064 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/a013d2208a1c40f5b5e7e91b0059589b 2024-11-26T15:25:45,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/30c38cafbf604047962862502c144e12 is 50, key is test_row_0/B:col10/1732634744234/Put/seqid=0 2024-11-26T15:25:45,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742025_1201 (size=12301) 2024-11-26T15:25:45,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T15:25:45,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:45,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:45,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:45,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T15:25:45,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:45,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:45,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:45,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634805393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:45,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634805412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:45,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634805412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:45,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634805424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:45,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:45,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:45,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:45,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T15:25:45,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:45,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:45,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:45,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/30c38cafbf604047962862502c144e12 2024-11-26T15:25:45,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/d4ef746d914645539c0b7e4e9d1a1b6f is 50, key is test_row_0/C:col10/1732634744234/Put/seqid=0 2024-11-26T15:25:45,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742026_1202 (size=12301) 2024-11-26T15:25:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T15:25:45,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T15:25:45,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:45,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:45,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:45,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T15:25:45,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:45,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:45,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:45,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/d4ef746d914645539c0b7e4e9d1a1b6f 2024-11-26T15:25:45,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/a013d2208a1c40f5b5e7e91b0059589b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/a013d2208a1c40f5b5e7e91b0059589b 2024-11-26T15:25:45,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/a013d2208a1c40f5b5e7e91b0059589b, entries=150, sequenceid=278, filesize=30.5 K 2024-11-26T15:25:45,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/30c38cafbf604047962862502c144e12 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/30c38cafbf604047962862502c144e12 2024-11-26T15:25:45,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T15:25:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:45,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:45,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:45,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:45,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/30c38cafbf604047962862502c144e12, entries=150, sequenceid=278, filesize=12.0 K 2024-11-26T15:25:45,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/d4ef746d914645539c0b7e4e9d1a1b6f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/d4ef746d914645539c0b7e4e9d1a1b6f 2024-11-26T15:25:45,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/d4ef746d914645539c0b7e4e9d1a1b6f, entries=150, sequenceid=278, filesize=12.0 K 2024-11-26T15:25:45,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 1679ms, sequenceid=278, compaction requested=false 2024-11-26T15:25:45,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:46,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T15:25:46,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:46,058 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-26T15:25:46,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c6b73e16c0bd4e698dcb48d87384cfaa_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634744252/Put/seqid=0 2024-11-26T15:25:46,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742027_1203 (size=12454) 2024-11-26T15:25:46,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:46,111 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c6b73e16c0bd4e698dcb48d87384cfaa_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c6b73e16c0bd4e698dcb48d87384cfaa_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/7ec7806fed6c4dfe95d4fa0b2b5942b3, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:46,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/7ec7806fed6c4dfe95d4fa0b2b5942b3 is 175, key is test_row_0/A:col10/1732634744252/Put/seqid=0 2024-11-26T15:25:46,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742028_1204 (size=31255) 2024-11-26T15:25:46,129 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/7ec7806fed6c4dfe95d4fa0b2b5942b3 2024-11-26T15:25:46,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/3ad493e7a61b4e9db19dfd7c09e17618 is 50, key is test_row_0/B:col10/1732634744252/Put/seqid=0 2024-11-26T15:25:46,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742029_1205 (size=12301) 2024-11-26T15:25:46,186 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/3ad493e7a61b4e9db19dfd7c09e17618 2024-11-26T15:25:46,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/d54ba2020aa54fe7bc9d9601a5c915cb is 50, key is test_row_0/C:col10/1732634744252/Put/seqid=0 2024-11-26T15:25:46,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742030_1206 (size=12301) 2024-11-26T15:25:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:46,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:46,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634806463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:46,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634806464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:46,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:46,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634806465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:46,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:46,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634806465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:46,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T15:25:46,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634806572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:46,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:46,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634806572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:46,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:46,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:46,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634806583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:46,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634806583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:46,677 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/d54ba2020aa54fe7bc9d9601a5c915cb 2024-11-26T15:25:46,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/7ec7806fed6c4dfe95d4fa0b2b5942b3 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7ec7806fed6c4dfe95d4fa0b2b5942b3 2024-11-26T15:25:46,696 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7ec7806fed6c4dfe95d4fa0b2b5942b3, entries=150, sequenceid=297, filesize=30.5 K 2024-11-26T15:25:46,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/3ad493e7a61b4e9db19dfd7c09e17618 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/3ad493e7a61b4e9db19dfd7c09e17618 2024-11-26T15:25:46,703 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/3ad493e7a61b4e9db19dfd7c09e17618, entries=150, sequenceid=297, filesize=12.0 K 2024-11-26T15:25:46,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/d54ba2020aa54fe7bc9d9601a5c915cb as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/d54ba2020aa54fe7bc9d9601a5c915cb 2024-11-26T15:25:46,710 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/d54ba2020aa54fe7bc9d9601a5c915cb, entries=150, sequenceid=297, filesize=12.0 K 2024-11-26T15:25:46,711 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 653ms, sequenceid=297, compaction requested=true 2024-11-26T15:25:46,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:46,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:46,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-26T15:25:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-26T15:25:46,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-26T15:25:46,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3380 sec 2024-11-26T15:25:46,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.3460 sec 2024-11-26T15:25:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:46,784 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:46,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-26T15:25:46,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bc84e226e319464da219611412374e3c_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634746464/Put/seqid=0 2024-11-26T15:25:46,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742031_1207 (size=12454) 2024-11-26T15:25:46,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634806827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:46,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634806829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634806831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:46,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634806833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:46,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:46,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:46,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634806942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634806943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634806943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634806943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:47,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634807153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:47,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:47,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634807153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:47,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634807153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:47,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634807153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:47,226 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:47,230 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bc84e226e319464da219611412374e3c_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bc84e226e319464da219611412374e3c_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:47,231 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/e81bf52595384f0c9a436a20bb92e417, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:47,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/e81bf52595384f0c9a436a20bb92e417 is 175, key is test_row_0/A:col10/1732634746464/Put/seqid=0 2024-11-26T15:25:47,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742032_1208 (size=31255) 2024-11-26T15:25:47,261 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/e81bf52595384f0c9a436a20bb92e417 2024-11-26T15:25:47,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/4392698de8da4d128b3643f0a27e6818 is 50, key is test_row_0/B:col10/1732634746464/Put/seqid=0 2024-11-26T15:25:47,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742033_1209 (size=12301) 2024-11-26T15:25:47,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/4392698de8da4d128b3643f0a27e6818 2024-11-26T15:25:47,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/0ceda11dec954a47aa4484641cce1105 is 50, key is test_row_0/C:col10/1732634746464/Put/seqid=0 2024-11-26T15:25:47,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742034_1210 (size=12301) 2024-11-26T15:25:47,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634807462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:47,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634807463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:47,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:47,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634807463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:47,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634807463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:47,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53676 deadline: 1732634807642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:47,648 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in 997d8d2a525d3ba7cd152ef78c94cfd4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:25:47,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:47,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:47,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/0ceda11dec954a47aa4484641cce1105 2024-11-26T15:25:47,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/e81bf52595384f0c9a436a20bb92e417 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e81bf52595384f0c9a436a20bb92e417 2024-11-26T15:25:47,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e81bf52595384f0c9a436a20bb92e417, entries=150, sequenceid=317, filesize=30.5 K 2024-11-26T15:25:47,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/4392698de8da4d128b3643f0a27e6818 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/4392698de8da4d128b3643f0a27e6818 2024-11-26T15:25:47,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/4392698de8da4d128b3643f0a27e6818, entries=150, sequenceid=317, filesize=12.0 K 2024-11-26T15:25:47,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/0ceda11dec954a47aa4484641cce1105 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/0ceda11dec954a47aa4484641cce1105 2024-11-26T15:25:47,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/0ceda11dec954a47aa4484641cce1105, entries=150, sequenceid=317, filesize=12.0 K 2024-11-26T15:25:47,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 1021ms, sequenceid=317, compaction requested=true 2024-11-26T15:25:47,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:47,805 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:25:47,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:47,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:47,806 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:25:47,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:47,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:47,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:47,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:47,809 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125534 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:25:47,809 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/A is initiating minor compaction (all files) 2024-11-26T15:25:47,809 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49718 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:25:47,809 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/A in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:47,809 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/B is initiating minor compaction (all files) 2024-11-26T15:25:47,809 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/B in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:47,809 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/1296fb951ea24db5a8df80fa68df6082, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/a013d2208a1c40f5b5e7e91b0059589b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7ec7806fed6c4dfe95d4fa0b2b5942b3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e81bf52595384f0c9a436a20bb92e417] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=122.6 K 2024-11-26T15:25:47,809 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/8556ad0c3a224bfe99b0605a93028b88, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/30c38cafbf604047962862502c144e12, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/3ad493e7a61b4e9db19dfd7c09e17618, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/4392698de8da4d128b3643f0a27e6818] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=48.6 K 2024-11-26T15:25:47,809 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:47,809 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/1296fb951ea24db5a8df80fa68df6082, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/a013d2208a1c40f5b5e7e91b0059589b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7ec7806fed6c4dfe95d4fa0b2b5942b3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e81bf52595384f0c9a436a20bb92e417] 2024-11-26T15:25:47,810 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1296fb951ea24db5a8df80fa68df6082, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732634742385 2024-11-26T15:25:47,810 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 8556ad0c3a224bfe99b0605a93028b88, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732634742385 2024-11-26T15:25:47,811 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 30c38cafbf604047962862502c144e12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732634743601 2024-11-26T15:25:47,811 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting a013d2208a1c40f5b5e7e91b0059589b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732634743601 2024-11-26T15:25:47,812 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ad493e7a61b4e9db19dfd7c09e17618, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732634744252 2024-11-26T15:25:47,812 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ec7806fed6c4dfe95d4fa0b2b5942b3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732634744252 2024-11-26T15:25:47,813 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 4392698de8da4d128b3643f0a27e6818, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732634746461 2024-11-26T15:25:47,813 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting e81bf52595384f0c9a436a20bb92e417, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732634746461 2024-11-26T15:25:47,825 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:47,826 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#B#compaction#173 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:47,827 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/7d695317d45341779c3a58f86753d827 is 50, key is test_row_0/B:col10/1732634746464/Put/seqid=0 2024-11-26T15:25:47,827 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126bbfaa495ce2a4afead34181ffff7aae2_997d8d2a525d3ba7cd152ef78c94cfd4 store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:47,829 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126bbfaa495ce2a4afead34181ffff7aae2_997d8d2a525d3ba7cd152ef78c94cfd4, store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:47,829 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bbfaa495ce2a4afead34181ffff7aae2_997d8d2a525d3ba7cd152ef78c94cfd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:47,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742035_1211 (size=13051) 2024-11-26T15:25:47,839 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/7d695317d45341779c3a58f86753d827 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/7d695317d45341779c3a58f86753d827 2024-11-26T15:25:47,845 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/B of 997d8d2a525d3ba7cd152ef78c94cfd4 into 7d695317d45341779c3a58f86753d827(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:47,846 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:47,846 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/B, priority=12, startTime=1732634747806; duration=0sec 2024-11-26T15:25:47,846 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:47,846 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:B 2024-11-26T15:25:47,846 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:25:47,847 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:25:47,847 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/C is initiating minor compaction (all files) 2024-11-26T15:25:47,847 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/C in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:47,847 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/c9b68488c3e5447e855b8fb9a704ff9f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/d4ef746d914645539c0b7e4e9d1a1b6f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/d54ba2020aa54fe7bc9d9601a5c915cb, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/0ceda11dec954a47aa4484641cce1105] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=48.5 K 2024-11-26T15:25:47,848 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting c9b68488c3e5447e855b8fb9a704ff9f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732634742385 2024-11-26T15:25:47,848 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting d4ef746d914645539c0b7e4e9d1a1b6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732634743601 2024-11-26T15:25:47,849 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting d54ba2020aa54fe7bc9d9601a5c915cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732634744252 2024-11-26T15:25:47,849 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ceda11dec954a47aa4484641cce1105, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732634746461 2024-11-26T15:25:47,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742036_1212 (size=4469) 2024-11-26T15:25:47,875 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#C#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:47,876 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/44bcb65c19c549dd8bfce1fbf84afedd is 50, key is test_row_0/C:col10/1732634746464/Put/seqid=0 2024-11-26T15:25:47,879 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#A#compaction#174 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:47,880 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/9e79be9ff40446b19f356aacf9a21d34 is 175, key is test_row_0/A:col10/1732634746464/Put/seqid=0 2024-11-26T15:25:47,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742038_1214 (size=32005) 2024-11-26T15:25:47,909 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/9e79be9ff40446b19f356aacf9a21d34 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/9e79be9ff40446b19f356aacf9a21d34 2024-11-26T15:25:47,916 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/A of 997d8d2a525d3ba7cd152ef78c94cfd4 into 9e79be9ff40446b19f356aacf9a21d34(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:47,916 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:47,916 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/A, priority=12, startTime=1732634747805; duration=0sec 2024-11-26T15:25:47,916 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:47,916 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:A 2024-11-26T15:25:47,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742037_1213 (size=13017) 2024-11-26T15:25:47,927 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/44bcb65c19c549dd8bfce1fbf84afedd as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/44bcb65c19c549dd8bfce1fbf84afedd 2024-11-26T15:25:47,935 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/C of 997d8d2a525d3ba7cd152ef78c94cfd4 into 44bcb65c19c549dd8bfce1fbf84afedd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:47,935 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:47,935 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/C, priority=12, startTime=1732634747806; duration=0sec 2024-11-26T15:25:47,935 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:47,935 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:C 2024-11-26T15:25:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:47,979 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:47,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-26T15:25:47,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112629b3b97a5d9a472793f10be979ea80a5_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634747973/Put/seqid=0 2024-11-26T15:25:48,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742039_1215 (size=12454) 2024-11-26T15:25:48,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:48,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634808003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634808003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:48,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:48,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:48,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634808007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:48,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634808007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:48,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:48,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634808113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:48,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634808113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:48,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634808122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634808122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:48,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:48,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634808322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:48,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634808322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:48,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634808332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:48,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:48,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634808333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:48,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:48,406 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:48,410 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112629b3b97a5d9a472793f10be979ea80a5_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112629b3b97a5d9a472793f10be979ea80a5_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:48,411 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/6791ab0a9e1147818d88b12eb398d4aa, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:48,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/6791ab0a9e1147818d88b12eb398d4aa is 175, key is test_row_0/A:col10/1732634747973/Put/seqid=0 2024-11-26T15:25:48,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742040_1216 (size=31255) 2024-11-26T15:25:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T15:25:48,542 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:48,543 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:25:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-26T15:25:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T15:25:48,544 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:25:48,545 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:25:48,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:25:48,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634808632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:48,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634808632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:48,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634808643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:48,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634808643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:48,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:48,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T15:25:48,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T15:25:48,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:48,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:48,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:48,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:48,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:48,816 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/6791ab0a9e1147818d88b12eb398d4aa 2024-11-26T15:25:48,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/d7e6fa3a4c224347a7c29e9ec2f49fc8 is 50, key is test_row_0/B:col10/1732634747973/Put/seqid=0 2024-11-26T15:25:48,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742041_1217 (size=12301) 2024-11-26T15:25:48,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T15:25:48,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:48,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:48,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:48,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:48,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T15:25:49,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T15:25:49,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:49,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:49,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:49,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:49,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:49,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:49,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:49,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634809142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:49,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:49,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634809143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:49,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634809153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:49,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634809153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:49,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T15:25:49,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:49,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:49,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:49,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:49,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:49,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:49,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:49,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:49,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:49,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T15:25:49,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/d7e6fa3a4c224347a7c29e9ec2f49fc8 2024-11-26T15:25:49,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/7851c44230c84489990b9ea8b85aac70 is 50, key is test_row_0/C:col10/1732634747973/Put/seqid=0 2024-11-26T15:25:49,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742042_1218 (size=12301) 2024-11-26T15:25:49,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/7851c44230c84489990b9ea8b85aac70 2024-11-26T15:25:49,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/6791ab0a9e1147818d88b12eb398d4aa as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/6791ab0a9e1147818d88b12eb398d4aa 2024-11-26T15:25:49,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/6791ab0a9e1147818d88b12eb398d4aa, entries=150, sequenceid=339, filesize=30.5 K 2024-11-26T15:25:49,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/d7e6fa3a4c224347a7c29e9ec2f49fc8 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/d7e6fa3a4c224347a7c29e9ec2f49fc8 2024-11-26T15:25:49,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/d7e6fa3a4c224347a7c29e9ec2f49fc8, entries=150, sequenceid=339, filesize=12.0 K 2024-11-26T15:25:49,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/7851c44230c84489990b9ea8b85aac70 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/7851c44230c84489990b9ea8b85aac70 2024-11-26T15:25:49,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/7851c44230c84489990b9ea8b85aac70, entries=150, sequenceid=339, filesize=12.0 K 2024-11-26T15:25:49,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 1300ms, sequenceid=339, compaction requested=false 2024-11-26T15:25:49,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:49,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T15:25:49,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:49,313 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-26T15:25:49,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112687eab6d7f1ba472896b4e4020209f837_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634748005/Put/seqid=0 2024-11-26T15:25:49,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742043_1219 (size=12454) 2024-11-26T15:25:49,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:49,334 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112687eab6d7f1ba472896b4e4020209f837_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112687eab6d7f1ba472896b4e4020209f837_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:49,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/1dccc91674cf4bc0bf4fbf8e29a20792, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:49,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/1dccc91674cf4bc0bf4fbf8e29a20792 is 175, key is test_row_0/A:col10/1732634748005/Put/seqid=0 2024-11-26T15:25:49,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742044_1220 (size=31255) 2024-11-26T15:25:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T15:25:49,739 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/1dccc91674cf4bc0bf4fbf8e29a20792 2024-11-26T15:25:49,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/373421f555664f52b5921a67255a79ba is 50, key is test_row_0/B:col10/1732634748005/Put/seqid=0 2024-11-26T15:25:49,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742045_1221 (size=12301) 2024-11-26T15:25:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:50,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:50,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634810174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634810176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634810177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634810178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,185 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/373421f555664f52b5921a67255a79ba 2024-11-26T15:25:50,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/2e0e12fe06df4cc3aba87ad3d0865dab is 50, key is test_row_0/C:col10/1732634748005/Put/seqid=0 2024-11-26T15:25:50,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742046_1222 (size=12301) 2024-11-26T15:25:50,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634810282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634810293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634810293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634810293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634810492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634810502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634810502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634810502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,605 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/2e0e12fe06df4cc3aba87ad3d0865dab 2024-11-26T15:25:50,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/1dccc91674cf4bc0bf4fbf8e29a20792 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/1dccc91674cf4bc0bf4fbf8e29a20792 2024-11-26T15:25:50,626 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/1dccc91674cf4bc0bf4fbf8e29a20792, entries=150, sequenceid=357, filesize=30.5 K 2024-11-26T15:25:50,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/373421f555664f52b5921a67255a79ba as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/373421f555664f52b5921a67255a79ba 2024-11-26T15:25:50,634 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/373421f555664f52b5921a67255a79ba, entries=150, sequenceid=357, filesize=12.0 K 2024-11-26T15:25:50,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/2e0e12fe06df4cc3aba87ad3d0865dab as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/2e0e12fe06df4cc3aba87ad3d0865dab 2024-11-26T15:25:50,646 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/2e0e12fe06df4cc3aba87ad3d0865dab, entries=150, sequenceid=357, filesize=12.0 K 2024-11-26T15:25:50,646 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 1333ms, sequenceid=357, compaction requested=true 2024-11-26T15:25:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-26T15:25:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-26T15:25:50,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-26T15:25:50,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1030 sec 2024-11-26T15:25:50,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.1080 sec 2024-11-26T15:25:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T15:25:50,703 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:50,703 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:25:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-26T15:25:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T15:25:50,705 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:25:50,707 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:25:50,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:25:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:50,806 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:50,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-26T15:25:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T15:25:50,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126777693ee3a7f4da2868a75169f1c9c66_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634750805/Put/seqid=0 2024-11-26T15:25:50,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742047_1223 (size=14994) 2024-11-26T15:25:50,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634810831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634810832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634810839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634810838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T15:25:50,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:50,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:50,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:50,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:50,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:50,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634810943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634810943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634810952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634810952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T15:25:51,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:51,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:51,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:51,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T15:25:51,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634811153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634811153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T15:25:51,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:51,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:51,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:51,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634811162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:51,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634811162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,219 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:51,223 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126777693ee3a7f4da2868a75169f1c9c66_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126777693ee3a7f4da2868a75169f1c9c66_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:51,226 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/58ead0acd8de45c5b77e602b53d96ac3, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:51,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/58ead0acd8de45c5b77e602b53d96ac3 is 175, key is test_row_0/A:col10/1732634750805/Put/seqid=0 2024-11-26T15:25:51,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742048_1224 (size=39949) 2024-11-26T15:25:51,232 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/58ead0acd8de45c5b77e602b53d96ac3 2024-11-26T15:25:51,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/36bf4c237539421ab13387d1d98a6cb3 is 50, key is test_row_0/B:col10/1732634750805/Put/seqid=0 2024-11-26T15:25:51,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742049_1225 (size=12301) 2024-11-26T15:25:51,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/36bf4c237539421ab13387d1d98a6cb3 2024-11-26T15:25:51,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/e3207f63935843c695953ea26ff1473f is 50, key is test_row_0/C:col10/1732634750805/Put/seqid=0 2024-11-26T15:25:51,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742050_1226 (size=12301) 2024-11-26T15:25:51,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/e3207f63935843c695953ea26ff1473f 2024-11-26T15:25:51,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/58ead0acd8de45c5b77e602b53d96ac3 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/58ead0acd8de45c5b77e602b53d96ac3 2024-11-26T15:25:51,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/58ead0acd8de45c5b77e602b53d96ac3, entries=200, sequenceid=376, filesize=39.0 K 2024-11-26T15:25:51,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/36bf4c237539421ab13387d1d98a6cb3 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/36bf4c237539421ab13387d1d98a6cb3 2024-11-26T15:25:51,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T15:25:51,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:51,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:51,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:51,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/36bf4c237539421ab13387d1d98a6cb3, entries=150, sequenceid=376, filesize=12.0 K 2024-11-26T15:25:51,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/e3207f63935843c695953ea26ff1473f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/e3207f63935843c695953ea26ff1473f 2024-11-26T15:25:51,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/e3207f63935843c695953ea26ff1473f, entries=150, sequenceid=376, filesize=12.0 K 2024-11-26T15:25:51,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 526ms, sequenceid=376, compaction requested=true 2024-11-26T15:25:51,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:51,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:25:51,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:51,332 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:25:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T15:25:51,332 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:25:51,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:25:51,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:51,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 997d8d2a525d3ba7cd152ef78c94cfd4:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:25:51,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:51,334 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:25:51,334 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/A is initiating minor compaction (all files) 2024-11-26T15:25:51,334 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/A in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:51,334 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/9e79be9ff40446b19f356aacf9a21d34, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/6791ab0a9e1147818d88b12eb398d4aa, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/1dccc91674cf4bc0bf4fbf8e29a20792, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/58ead0acd8de45c5b77e602b53d96ac3] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=131.3 K 2024-11-26T15:25:51,334 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:51,334 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/9e79be9ff40446b19f356aacf9a21d34, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/6791ab0a9e1147818d88b12eb398d4aa, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/1dccc91674cf4bc0bf4fbf8e29a20792, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/58ead0acd8de45c5b77e602b53d96ac3] 2024-11-26T15:25:51,335 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e79be9ff40446b19f356aacf9a21d34, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732634746461 2024-11-26T15:25:51,335 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6791ab0a9e1147818d88b12eb398d4aa, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732634747973 2024-11-26T15:25:51,335 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dccc91674cf4bc0bf4fbf8e29a20792, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732634747999 2024-11-26T15:25:51,336 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58ead0acd8de45c5b77e602b53d96ac3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732634750172 2024-11-26T15:25:51,336 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:25:51,336 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/B is initiating minor compaction (all files) 2024-11-26T15:25:51,336 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/B in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:51,336 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/7d695317d45341779c3a58f86753d827, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/d7e6fa3a4c224347a7c29e9ec2f49fc8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/373421f555664f52b5921a67255a79ba, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/36bf4c237539421ab13387d1d98a6cb3] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=48.8 K 2024-11-26T15:25:51,337 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d695317d45341779c3a58f86753d827, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732634746461 2024-11-26T15:25:51,337 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting d7e6fa3a4c224347a7c29e9ec2f49fc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732634747973 2024-11-26T15:25:51,337 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 373421f555664f52b5921a67255a79ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732634747999 2024-11-26T15:25:51,338 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 36bf4c237539421ab13387d1d98a6cb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732634750176 2024-11-26T15:25:51,352 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:51,358 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#B#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:51,359 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/ec76a0bba2c54c43bdf8d235ab5c4c6a is 50, key is test_row_0/B:col10/1732634750805/Put/seqid=0 2024-11-26T15:25:51,360 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411262e31c726412245bda94411ea01fb2baa_997d8d2a525d3ba7cd152ef78c94cfd4 store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:51,361 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411262e31c726412245bda94411ea01fb2baa_997d8d2a525d3ba7cd152ef78c94cfd4, store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:51,361 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262e31c726412245bda94411ea01fb2baa_997d8d2a525d3ba7cd152ef78c94cfd4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:51,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742052_1228 (size=4469) 2024-11-26T15:25:51,374 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#A#compaction#185 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:51,375 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/0087f912bc4b4a138aba13c3dcabf229 is 175, key is test_row_0/A:col10/1732634750805/Put/seqid=0 2024-11-26T15:25:51,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742051_1227 (size=13187) 2024-11-26T15:25:51,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742053_1229 (size=32141) 2024-11-26T15:25:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:51,465 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:25:51,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-26T15:25:51,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T15:25:51,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:51,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:51,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:51,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634811487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:51,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:51,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634811488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:51,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634811489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634811490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260085bfb6cb704622aece5b3daee8a077_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634751464/Put/seqid=0 2024-11-26T15:25:51,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742054_1230 (size=17534) 2024-11-26T15:25:51,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:51,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634811602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634811602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634811602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634811603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:51,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T15:25:51,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:51,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:51,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:51,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,757 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-26T15:25:51,757 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:51,757 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:51,757 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:51,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:51,758 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-26T15:25:51,758 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:51,758 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:51,758 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:51,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:51,759 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-11-26T15:25:51,759 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:51,759 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:51,759 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:51,759 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:51,761 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-26T15:25:51,761 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:51,761 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:51,761 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:51,761 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:51,796 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/ec76a0bba2c54c43bdf8d235ab5c4c6a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/ec76a0bba2c54c43bdf8d235ab5c4c6a 2024-11-26T15:25:51,803 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/B of 997d8d2a525d3ba7cd152ef78c94cfd4 into ec76a0bba2c54c43bdf8d235ab5c4c6a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:51,804 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:51,804 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/B, priority=12, startTime=1732634751332; duration=0sec 2024-11-26T15:25:51,804 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:25:51,804 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:B 2024-11-26T15:25:51,804 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:25:51,804 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/0087f912bc4b4a138aba13c3dcabf229 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/0087f912bc4b4a138aba13c3dcabf229 2024-11-26T15:25:51,805 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:25:51,805 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 997d8d2a525d3ba7cd152ef78c94cfd4/C is initiating minor compaction (all files) 2024-11-26T15:25:51,805 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 997d8d2a525d3ba7cd152ef78c94cfd4/C in TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:51,805 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/44bcb65c19c549dd8bfce1fbf84afedd, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/7851c44230c84489990b9ea8b85aac70, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/2e0e12fe06df4cc3aba87ad3d0865dab, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/e3207f63935843c695953ea26ff1473f] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp, totalSize=48.8 K 2024-11-26T15:25:51,806 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 44bcb65c19c549dd8bfce1fbf84afedd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732634746461 2024-11-26T15:25:51,806 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 7851c44230c84489990b9ea8b85aac70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732634747973 2024-11-26T15:25:51,806 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e0e12fe06df4cc3aba87ad3d0865dab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732634747999 2024-11-26T15:25:51,806 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting e3207f63935843c695953ea26ff1473f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732634750176 2024-11-26T15:25:51,809 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/A of 997d8d2a525d3ba7cd152ef78c94cfd4 into 0087f912bc4b4a138aba13c3dcabf229(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:51,809 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:51,809 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/A, priority=12, startTime=1732634751332; duration=0sec 2024-11-26T15:25:51,809 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:51,809 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:A 2024-11-26T15:25:51,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:51,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:51,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634811813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634811812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:51,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:51,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634811813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:51,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,814 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 997d8d2a525d3ba7cd152ef78c94cfd4#C#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:25:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634811813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:51,814 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/f3139d4e02fd4b93aede76f32ee6999a is 50, key is test_row_0/C:col10/1732634750805/Put/seqid=0 2024-11-26T15:25:51,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:51,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:51,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742055_1231 (size=13153) 2024-11-26T15:25:51,826 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/f3139d4e02fd4b93aede76f32ee6999a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/f3139d4e02fd4b93aede76f32ee6999a 2024-11-26T15:25:51,832 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 997d8d2a525d3ba7cd152ef78c94cfd4/C of 997d8d2a525d3ba7cd152ef78c94cfd4 into f3139d4e02fd4b93aede76f32ee6999a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:25:51,832 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:51,832 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., storeName=997d8d2a525d3ba7cd152ef78c94cfd4/C, priority=12, startTime=1732634751333; duration=0sec 2024-11-26T15:25:51,833 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:25:51,833 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 997d8d2a525d3ba7cd152ef78c94cfd4:C 2024-11-26T15:25:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T15:25:51,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T15:25:51,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:51,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:51,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:51,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:51,955 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:51,959 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260085bfb6cb704622aece5b3daee8a077_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260085bfb6cb704622aece5b3daee8a077_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:51,960 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/2170043f08274c75a3b4a459f2b0d7ad, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:51,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/2170043f08274c75a3b4a459f2b0d7ad is 175, key is test_row_0/A:col10/1732634751464/Put/seqid=0 2024-11-26T15:25:51,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742056_1232 (size=48639) 2024-11-26T15:25:52,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T15:25:52,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:52,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:52,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:52,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:52,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634812122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:52,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634812123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634812123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:52,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634812123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:52,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T15:25:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:52,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T15:25:52,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:52,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:52,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:52,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,365 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/2170043f08274c75a3b4a459f2b0d7ad 2024-11-26T15:25:52,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/85c3dca37f7747149aff45e450f9882f is 50, key is test_row_0/B:col10/1732634751464/Put/seqid=0 2024-11-26T15:25:52,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742057_1233 (size=12301) 2024-11-26T15:25:52,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T15:25:52,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:52,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:52,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:52,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T15:25:52,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:52,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:52,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:52,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:52,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53684 deadline: 1732634812632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53648 deadline: 1732634812633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:52,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53662 deadline: 1732634812633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:52,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:25:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53638 deadline: 1732634812634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:52,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:52,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:52,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:52,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:52,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:25:52,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:25:52,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T15:25:52,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:52,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:52,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:52,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/85c3dca37f7747149aff45e450f9882f 2024-11-26T15:25:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/ee9cff97dee749a8bf21e57255b0e394 is 50, key is test_row_0/C:col10/1732634751464/Put/seqid=0 2024-11-26T15:25:52,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742058_1234 (size=12301) 2024-11-26T15:25:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T15:25:52,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T15:25:52,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:52,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:52,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:52,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:53,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T15:25:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:53,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:53,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:25:53,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/ee9cff97dee749a8bf21e57255b0e394 2024-11-26T15:25:53,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/2170043f08274c75a3b4a459f2b0d7ad as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/2170043f08274c75a3b4a459f2b0d7ad 2024-11-26T15:25:53,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/2170043f08274c75a3b4a459f2b0d7ad, entries=250, sequenceid=395, filesize=47.5 K 2024-11-26T15:25:53,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/85c3dca37f7747149aff45e450f9882f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/85c3dca37f7747149aff45e450f9882f 2024-11-26T15:25:53,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/85c3dca37f7747149aff45e450f9882f, entries=150, sequenceid=395, filesize=12.0 K 2024-11-26T15:25:53,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/ee9cff97dee749a8bf21e57255b0e394 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/ee9cff97dee749a8bf21e57255b0e394 2024-11-26T15:25:53,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/ee9cff97dee749a8bf21e57255b0e394, entries=150, sequenceid=395, filesize=12.0 K 2024-11-26T15:25:53,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 1750ms, sequenceid=395, compaction requested=false 2024-11-26T15:25:53,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:53,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T15:25:53,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:53,237 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-26T15:25:53,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265f1a1bec39294ffbbfdec58208f717c8_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634751489/Put/seqid=0 2024-11-26T15:25:53,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742059_1235 (size=12454) 2024-11-26T15:25:53,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:53,252 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265f1a1bec39294ffbbfdec58208f717c8_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f1a1bec39294ffbbfdec58208f717c8_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:53,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/7d37d9be447b4994841ec00648684774, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:53,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/7d37d9be447b4994841ec00648684774 is 175, key is test_row_0/A:col10/1732634751489/Put/seqid=0 2024-11-26T15:25:53,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742060_1236 (size=31255) 2024-11-26T15:25:53,258 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=415, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/7d37d9be447b4994841ec00648684774 2024-11-26T15:25:53,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/bbf81d2779cb47bbb688cd6bbac654de is 50, key is test_row_0/B:col10/1732634751489/Put/seqid=0 2024-11-26T15:25:53,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742061_1237 (size=12301) 2024-11-26T15:25:53,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:53,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. as already flushing 2024-11-26T15:25:53,646 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-26T15:25:53,646 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:53,646 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:53,647 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:53,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:53,656 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-11-26T15:25:53,656 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:53,656 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:53,656 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:53,656 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:53,656 INFO [Thread-630 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-630. 2024-11-26T15:25:53,657 DEBUG [Thread-630 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:53,657 INFO [Thread-628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-628. 2024-11-26T15:25:53,657 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:53,657 DEBUG [Thread-628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:53,657 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:53,657 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:53,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:53,657 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:53,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:53,669 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/bbf81d2779cb47bbb688cd6bbac654de 2024-11-26T15:25:53,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/341a61b9a0be4a829df52e9ddfb19d1e is 50, key is test_row_0/C:col10/1732634751489/Put/seqid=0 2024-11-26T15:25:53,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742062_1238 (size=12301) 2024-11-26T15:25:54,086 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/341a61b9a0be4a829df52e9ddfb19d1e 2024-11-26T15:25:54,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/7d37d9be447b4994841ec00648684774 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7d37d9be447b4994841ec00648684774 2024-11-26T15:25:54,094 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7d37d9be447b4994841ec00648684774, entries=150, sequenceid=415, filesize=30.5 K 2024-11-26T15:25:54,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/bbf81d2779cb47bbb688cd6bbac654de as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/bbf81d2779cb47bbb688cd6bbac654de 2024-11-26T15:25:54,100 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/bbf81d2779cb47bbb688cd6bbac654de, entries=150, sequenceid=415, filesize=12.0 K 2024-11-26T15:25:54,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/341a61b9a0be4a829df52e9ddfb19d1e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/341a61b9a0be4a829df52e9ddfb19d1e 2024-11-26T15:25:54,105 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/341a61b9a0be4a829df52e9ddfb19d1e, entries=150, sequenceid=415, filesize=12.0 K 2024-11-26T15:25:54,105 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=26.84 KB/27480 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 868ms, sequenceid=415, compaction requested=true 2024-11-26T15:25:54,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 997d8d2a525d3ba7cd152ef78c94cfd4: 2024-11-26T15:25:54,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:54,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-26T15:25:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-26T15:25:54,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-26T15:25:54,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4000 sec 2024-11-26T15:25:54,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 3.4070 sec 2024-11-26T15:25:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T15:25:54,875 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:55,237 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T15:25:57,668 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-11-26T15:25:57,668 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:25:57,668 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:57,668 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:57,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:57,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T15:25:57,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-26T15:25:57,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-26T15:25:57,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-26T15:25:57,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-26T15:25:57,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-26T15:25:57,669 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T15:25:57,670 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3641 2024-11-26T15:25:57,670 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3683 2024-11-26T15:25:57,670 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T15:25:57,670 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1611 2024-11-26T15:25:57,670 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4833 rows 2024-11-26T15:25:57,670 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1606 2024-11-26T15:25:57,670 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4818 rows 2024-11-26T15:25:57,670 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T15:25:57,670 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T15:25:57,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:57,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:25:57,671 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:25:57,672 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T15:25:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T15:25:57,677 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634757677"}]},"ts":"1732634757677"} 2024-11-26T15:25:57,680 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T15:25:57,680 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T15:25:57,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T15:25:57,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=997d8d2a525d3ba7cd152ef78c94cfd4, UNASSIGN}] 2024-11-26T15:25:57,683 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=997d8d2a525d3ba7cd152ef78c94cfd4, UNASSIGN 2024-11-26T15:25:57,683 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=997d8d2a525d3ba7cd152ef78c94cfd4, regionState=CLOSING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:25:57,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=997d8d2a525d3ba7cd152ef78c94cfd4, UNASSIGN because future has completed 2024-11-26T15:25:57,685 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T15:25:57,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:25:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T15:25:57,838 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:57,838 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T15:25:57,838 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing 997d8d2a525d3ba7cd152ef78c94cfd4, disabling compactions & flushes 2024-11-26T15:25:57,838 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:57,838 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:57,838 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. after waiting 0 ms 2024-11-26T15:25:57,838 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:57,839 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing 997d8d2a525d3ba7cd152ef78c94cfd4 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-26T15:25:57,851 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112635af25d66cb34c1c9e3cf27fce3b7a43_997d8d2a525d3ba7cd152ef78c94cfd4 is 50, key is test_row_0/A:col10/1732634757664/Put/seqid=0 2024-11-26T15:25:57,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742063_1239 (size=12454) 2024-11-26T15:25:57,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T15:25:58,257 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:58,268 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112635af25d66cb34c1c9e3cf27fce3b7a43_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112635af25d66cb34c1c9e3cf27fce3b7a43_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:58,269 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/11672751f0344fb9821bc28c13ba7e75, store: [table=TestAcidGuarantees family=A region=997d8d2a525d3ba7cd152ef78c94cfd4] 2024-11-26T15:25:58,270 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/11672751f0344fb9821bc28c13ba7e75 is 175, key is test_row_0/A:col10/1732634757664/Put/seqid=0 2024-11-26T15:25:58,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742064_1240 (size=31255) 2024-11-26T15:25:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T15:25:58,676 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/11672751f0344fb9821bc28c13ba7e75 2024-11-26T15:25:58,689 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/caf59a066e0045bcb8b0a0e5b9f79921 is 50, key is test_row_0/B:col10/1732634757664/Put/seqid=0 2024-11-26T15:25:58,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742065_1241 (size=12301) 2024-11-26T15:25:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T15:25:59,096 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/caf59a066e0045bcb8b0a0e5b9f79921 2024-11-26T15:25:59,110 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/d811cd9e403e4259afa523e1c7afb101 is 50, key is test_row_0/C:col10/1732634757664/Put/seqid=0 2024-11-26T15:25:59,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742066_1242 (size=12301) 2024-11-26T15:25:59,516 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/d811cd9e403e4259afa523e1c7afb101 2024-11-26T15:25:59,525 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/A/11672751f0344fb9821bc28c13ba7e75 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/11672751f0344fb9821bc28c13ba7e75 2024-11-26T15:25:59,529 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/11672751f0344fb9821bc28c13ba7e75, entries=150, sequenceid=423, filesize=30.5 K 2024-11-26T15:25:59,530 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/B/caf59a066e0045bcb8b0a0e5b9f79921 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/caf59a066e0045bcb8b0a0e5b9f79921 2024-11-26T15:25:59,534 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/caf59a066e0045bcb8b0a0e5b9f79921, entries=150, sequenceid=423, filesize=12.0 K 2024-11-26T15:25:59,535 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/.tmp/C/d811cd9e403e4259afa523e1c7afb101 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/d811cd9e403e4259afa523e1c7afb101 2024-11-26T15:25:59,540 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/d811cd9e403e4259afa523e1c7afb101, entries=150, sequenceid=423, filesize=12.0 K 2024-11-26T15:25:59,541 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 1703ms, sequenceid=423, compaction requested=true 2024-11-26T15:25:59,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/66ae5c4df92944fca22cbab39adfd4e4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/de18ba24690b483395e789038c0e68cb, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/26e17c901b9b4171be0b169241d3e08d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/087fd1afacc5472cbec25a2fe7797aa7, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/3960bc94348d447b9e1c0007d5a87751, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/a8034cc6fc6f474fa3c7f25714046593, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/4e109fda2c694cb9a9786aaf54db5f89, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e3b820b8174b44438f69defacd138ce6, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7736efe8ecb447059e480b71f1c19dd2, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/14c9474da0154333a0b7849f135efe20, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e0170a8d52704a5d9e02ed9cb1e495c0, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/8892579a190f4dc4a1972c99f8296bfe, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7cc6dd53676a47a397ad08061523258f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/fb9b020ef6924697ba014a153948d83f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/f7539088c9664e45a7cadc19518156fb, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/26e15d9dff814b32a99fc40f4b51fd7c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/d9b530f3856c492c8094cfbac57522be, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7db95863f44a4393b2aaafd811fc0179, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/1296fb951ea24db5a8df80fa68df6082, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/a013d2208a1c40f5b5e7e91b0059589b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7ec7806fed6c4dfe95d4fa0b2b5942b3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/9e79be9ff40446b19f356aacf9a21d34, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e81bf52595384f0c9a436a20bb92e417, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/6791ab0a9e1147818d88b12eb398d4aa, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/1dccc91674cf4bc0bf4fbf8e29a20792, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/58ead0acd8de45c5b77e602b53d96ac3] to archive 2024-11-26T15:25:59,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:25:59,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/66ae5c4df92944fca22cbab39adfd4e4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/66ae5c4df92944fca22cbab39adfd4e4 2024-11-26T15:25:59,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/de18ba24690b483395e789038c0e68cb to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/de18ba24690b483395e789038c0e68cb 2024-11-26T15:25:59,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/26e17c901b9b4171be0b169241d3e08d to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/26e17c901b9b4171be0b169241d3e08d 2024-11-26T15:25:59,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/087fd1afacc5472cbec25a2fe7797aa7 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/087fd1afacc5472cbec25a2fe7797aa7 2024-11-26T15:25:59,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/3960bc94348d447b9e1c0007d5a87751 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/3960bc94348d447b9e1c0007d5a87751 2024-11-26T15:25:59,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/a8034cc6fc6f474fa3c7f25714046593 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/a8034cc6fc6f474fa3c7f25714046593 2024-11-26T15:25:59,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/4e109fda2c694cb9a9786aaf54db5f89 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/4e109fda2c694cb9a9786aaf54db5f89 2024-11-26T15:25:59,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e3b820b8174b44438f69defacd138ce6 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e3b820b8174b44438f69defacd138ce6 2024-11-26T15:25:59,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7736efe8ecb447059e480b71f1c19dd2 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7736efe8ecb447059e480b71f1c19dd2 2024-11-26T15:25:59,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/14c9474da0154333a0b7849f135efe20 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/14c9474da0154333a0b7849f135efe20 2024-11-26T15:25:59,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e0170a8d52704a5d9e02ed9cb1e495c0 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e0170a8d52704a5d9e02ed9cb1e495c0 2024-11-26T15:25:59,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/8892579a190f4dc4a1972c99f8296bfe to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/8892579a190f4dc4a1972c99f8296bfe 2024-11-26T15:25:59,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7cc6dd53676a47a397ad08061523258f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7cc6dd53676a47a397ad08061523258f 2024-11-26T15:25:59,562 DEBUG [regionserver/6cc2bc8bf733:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-26T15:25:59,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/fb9b020ef6924697ba014a153948d83f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/fb9b020ef6924697ba014a153948d83f 2024-11-26T15:25:59,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/f7539088c9664e45a7cadc19518156fb to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/f7539088c9664e45a7cadc19518156fb 2024-11-26T15:25:59,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/26e15d9dff814b32a99fc40f4b51fd7c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/26e15d9dff814b32a99fc40f4b51fd7c 2024-11-26T15:25:59,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/d9b530f3856c492c8094cfbac57522be to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/d9b530f3856c492c8094cfbac57522be 2024-11-26T15:25:59,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7db95863f44a4393b2aaafd811fc0179 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7db95863f44a4393b2aaafd811fc0179 2024-11-26T15:25:59,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/1296fb951ea24db5a8df80fa68df6082 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/1296fb951ea24db5a8df80fa68df6082 2024-11-26T15:25:59,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/a013d2208a1c40f5b5e7e91b0059589b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/a013d2208a1c40f5b5e7e91b0059589b 2024-11-26T15:25:59,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7ec7806fed6c4dfe95d4fa0b2b5942b3 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7ec7806fed6c4dfe95d4fa0b2b5942b3 2024-11-26T15:25:59,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/9e79be9ff40446b19f356aacf9a21d34 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/9e79be9ff40446b19f356aacf9a21d34 2024-11-26T15:25:59,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e81bf52595384f0c9a436a20bb92e417 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/e81bf52595384f0c9a436a20bb92e417 2024-11-26T15:25:59,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/6791ab0a9e1147818d88b12eb398d4aa to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/6791ab0a9e1147818d88b12eb398d4aa 2024-11-26T15:25:59,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/1dccc91674cf4bc0bf4fbf8e29a20792 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/1dccc91674cf4bc0bf4fbf8e29a20792 2024-11-26T15:25:59,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/58ead0acd8de45c5b77e602b53d96ac3 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/58ead0acd8de45c5b77e602b53d96ac3 2024-11-26T15:25:59,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/ce27236042234386a59319b3abc6d9ef, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/6201e4fa90104fc7819b6f7c3cc3d7c4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/ba3c99c0840a49b7ad504958716843bb, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/b6e1bf7be74746baae5f5bd9c70cd373, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/a9ef100e1dbc4c58a596cf42196643c1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/43023c337d0f44d6a4f17c15f41b8966, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/e2109e5cf6ef46e9b6f27f46a859b914, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/c83df15cf4fe45a6952b9b19fcea1abf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/f5daf56ed73c428e864a9d60951dcd98, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/8a391479ce8b47579917f930eb9e2745, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/9c6c7f4530ea44d59cb521b06d4ef267, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/cc6b24fd9f5d4e199212d2c5fe15a7c9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/df10ad4b6b224daab3d5a2382ec33cf1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/5904a4d5697049eca107617adb042ddf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/0968d372a4d24709a2d8704ef8750c1e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/97a717a1aa7341ca98fe3bb4ae719982, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/28d6bbac95584b929db9dbf770cf8365, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/8556ad0c3a224bfe99b0605a93028b88, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/31ba4758b26a4336bee3721341b2f6d5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/30c38cafbf604047962862502c144e12, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/3ad493e7a61b4e9db19dfd7c09e17618, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/7d695317d45341779c3a58f86753d827, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/4392698de8da4d128b3643f0a27e6818, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/d7e6fa3a4c224347a7c29e9ec2f49fc8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/373421f555664f52b5921a67255a79ba, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/36bf4c237539421ab13387d1d98a6cb3] to archive 2024-11-26T15:25:59,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:25:59,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/ce27236042234386a59319b3abc6d9ef to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/ce27236042234386a59319b3abc6d9ef 2024-11-26T15:25:59,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/6201e4fa90104fc7819b6f7c3cc3d7c4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/6201e4fa90104fc7819b6f7c3cc3d7c4 2024-11-26T15:25:59,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/ba3c99c0840a49b7ad504958716843bb to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/ba3c99c0840a49b7ad504958716843bb 2024-11-26T15:25:59,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/b6e1bf7be74746baae5f5bd9c70cd373 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/b6e1bf7be74746baae5f5bd9c70cd373 2024-11-26T15:25:59,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/a9ef100e1dbc4c58a596cf42196643c1 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/a9ef100e1dbc4c58a596cf42196643c1 2024-11-26T15:25:59,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/43023c337d0f44d6a4f17c15f41b8966 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/43023c337d0f44d6a4f17c15f41b8966 2024-11-26T15:25:59,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/e2109e5cf6ef46e9b6f27f46a859b914 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/e2109e5cf6ef46e9b6f27f46a859b914 2024-11-26T15:25:59,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/c83df15cf4fe45a6952b9b19fcea1abf to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/c83df15cf4fe45a6952b9b19fcea1abf 2024-11-26T15:25:59,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/f5daf56ed73c428e864a9d60951dcd98 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/f5daf56ed73c428e864a9d60951dcd98 2024-11-26T15:25:59,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/8a391479ce8b47579917f930eb9e2745 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/8a391479ce8b47579917f930eb9e2745 2024-11-26T15:25:59,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/9c6c7f4530ea44d59cb521b06d4ef267 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/9c6c7f4530ea44d59cb521b06d4ef267 2024-11-26T15:25:59,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/cc6b24fd9f5d4e199212d2c5fe15a7c9 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/cc6b24fd9f5d4e199212d2c5fe15a7c9 2024-11-26T15:25:59,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/df10ad4b6b224daab3d5a2382ec33cf1 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/df10ad4b6b224daab3d5a2382ec33cf1 2024-11-26T15:25:59,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/5904a4d5697049eca107617adb042ddf to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/5904a4d5697049eca107617adb042ddf 2024-11-26T15:25:59,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/0968d372a4d24709a2d8704ef8750c1e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/0968d372a4d24709a2d8704ef8750c1e 2024-11-26T15:25:59,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/97a717a1aa7341ca98fe3bb4ae719982 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/97a717a1aa7341ca98fe3bb4ae719982 2024-11-26T15:25:59,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/28d6bbac95584b929db9dbf770cf8365 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/28d6bbac95584b929db9dbf770cf8365 2024-11-26T15:25:59,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/8556ad0c3a224bfe99b0605a93028b88 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/8556ad0c3a224bfe99b0605a93028b88 2024-11-26T15:25:59,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/31ba4758b26a4336bee3721341b2f6d5 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/31ba4758b26a4336bee3721341b2f6d5 2024-11-26T15:25:59,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/30c38cafbf604047962862502c144e12 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/30c38cafbf604047962862502c144e12 2024-11-26T15:25:59,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/3ad493e7a61b4e9db19dfd7c09e17618 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/3ad493e7a61b4e9db19dfd7c09e17618 2024-11-26T15:25:59,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/7d695317d45341779c3a58f86753d827 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/7d695317d45341779c3a58f86753d827 2024-11-26T15:25:59,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/4392698de8da4d128b3643f0a27e6818 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/4392698de8da4d128b3643f0a27e6818 2024-11-26T15:25:59,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/d7e6fa3a4c224347a7c29e9ec2f49fc8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/d7e6fa3a4c224347a7c29e9ec2f49fc8 2024-11-26T15:25:59,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/373421f555664f52b5921a67255a79ba to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/373421f555664f52b5921a67255a79ba 2024-11-26T15:25:59,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/36bf4c237539421ab13387d1d98a6cb3 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/36bf4c237539421ab13387d1d98a6cb3 2024-11-26T15:25:59,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/60b68554821e4cc198852cf5a5e4e295, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/8caaf450f3c54d6591b0e56d1c166235, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/72efeee9270347f1b94f898660157f38, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/fce41aa7e14b47e89d517be201b0dea8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/ef143ead8e3d45a79ab25d2101e1b1c5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/9947436b861b4109a598e901f55f80ee, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/5aeeeeb72188475599213886f786f183, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/8626f6982ef9487295f6d0cbb2824183, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/b36d5a925b994866b8a04bbe8000076a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/72c317b00c4e45e393348098426da0c6, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/fb6eaecee31342919361dad29bf38a21, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/27992ebc6ab944ea81fe57497eeb51df, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/b8716d63e2d34c54b15347bb35edc1db, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/95019308e1774a8d852b915ace612b2b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/bd0e337b73bd4f508ae5817e635a5276, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/4a81788276cf4da2848a88bc242846be, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/c9b68488c3e5447e855b8fb9a704ff9f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/f30093d02bd3414c90977d234b292193, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/d4ef746d914645539c0b7e4e9d1a1b6f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/d54ba2020aa54fe7bc9d9601a5c915cb, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/44bcb65c19c549dd8bfce1fbf84afedd, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/0ceda11dec954a47aa4484641cce1105, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/7851c44230c84489990b9ea8b85aac70, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/2e0e12fe06df4cc3aba87ad3d0865dab, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/e3207f63935843c695953ea26ff1473f] to archive 2024-11-26T15:25:59,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:25:59,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/60b68554821e4cc198852cf5a5e4e295 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/60b68554821e4cc198852cf5a5e4e295 2024-11-26T15:25:59,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/8caaf450f3c54d6591b0e56d1c166235 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/8caaf450f3c54d6591b0e56d1c166235 2024-11-26T15:25:59,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/72efeee9270347f1b94f898660157f38 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/72efeee9270347f1b94f898660157f38 2024-11-26T15:25:59,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/fce41aa7e14b47e89d517be201b0dea8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/fce41aa7e14b47e89d517be201b0dea8 2024-11-26T15:25:59,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/ef143ead8e3d45a79ab25d2101e1b1c5 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/ef143ead8e3d45a79ab25d2101e1b1c5 2024-11-26T15:25:59,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/9947436b861b4109a598e901f55f80ee to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/9947436b861b4109a598e901f55f80ee 2024-11-26T15:25:59,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/5aeeeeb72188475599213886f786f183 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/5aeeeeb72188475599213886f786f183 2024-11-26T15:25:59,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/8626f6982ef9487295f6d0cbb2824183 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/8626f6982ef9487295f6d0cbb2824183 2024-11-26T15:25:59,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/b36d5a925b994866b8a04bbe8000076a to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/b36d5a925b994866b8a04bbe8000076a 2024-11-26T15:25:59,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/72c317b00c4e45e393348098426da0c6 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/72c317b00c4e45e393348098426da0c6 2024-11-26T15:25:59,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/fb6eaecee31342919361dad29bf38a21 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/fb6eaecee31342919361dad29bf38a21 2024-11-26T15:25:59,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/27992ebc6ab944ea81fe57497eeb51df to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/27992ebc6ab944ea81fe57497eeb51df 2024-11-26T15:25:59,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/b8716d63e2d34c54b15347bb35edc1db to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/b8716d63e2d34c54b15347bb35edc1db 2024-11-26T15:25:59,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/95019308e1774a8d852b915ace612b2b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/95019308e1774a8d852b915ace612b2b 2024-11-26T15:25:59,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/bd0e337b73bd4f508ae5817e635a5276 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/bd0e337b73bd4f508ae5817e635a5276 2024-11-26T15:25:59,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/4a81788276cf4da2848a88bc242846be to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/4a81788276cf4da2848a88bc242846be 2024-11-26T15:25:59,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/c9b68488c3e5447e855b8fb9a704ff9f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/c9b68488c3e5447e855b8fb9a704ff9f 2024-11-26T15:25:59,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/f30093d02bd3414c90977d234b292193 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/f30093d02bd3414c90977d234b292193 2024-11-26T15:25:59,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/d4ef746d914645539c0b7e4e9d1a1b6f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/d4ef746d914645539c0b7e4e9d1a1b6f 2024-11-26T15:25:59,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/d54ba2020aa54fe7bc9d9601a5c915cb to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/d54ba2020aa54fe7bc9d9601a5c915cb 2024-11-26T15:25:59,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/44bcb65c19c549dd8bfce1fbf84afedd to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/44bcb65c19c549dd8bfce1fbf84afedd 2024-11-26T15:25:59,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/0ceda11dec954a47aa4484641cce1105 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/0ceda11dec954a47aa4484641cce1105 2024-11-26T15:25:59,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/7851c44230c84489990b9ea8b85aac70 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/7851c44230c84489990b9ea8b85aac70 2024-11-26T15:25:59,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/2e0e12fe06df4cc3aba87ad3d0865dab to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/2e0e12fe06df4cc3aba87ad3d0865dab 2024-11-26T15:25:59,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/e3207f63935843c695953ea26ff1473f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/e3207f63935843c695953ea26ff1473f 2024-11-26T15:25:59,629 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/recovered.edits/426.seqid, newMaxSeqId=426, maxSeqId=4 2024-11-26T15:25:59,629 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. 2024-11-26T15:25:59,629 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for 997d8d2a525d3ba7cd152ef78c94cfd4: Waiting for close lock at 1732634757838Running coprocessor pre-close hooks at 1732634757838Disabling compacts and flushes for region at 1732634757838Disabling writes for close at 1732634757838Obtaining lock to block concurrent updates at 1732634757839 (+1 ms)Preparing flush snapshotting stores in 997d8d2a525d3ba7cd152ef78c94cfd4 at 1732634757839Finished memstore snapshotting TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732634757839Flushing stores of TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4. at 1732634757840 (+1 ms)Flushing 997d8d2a525d3ba7cd152ef78c94cfd4/A: creating writer at 1732634757841 (+1 ms)Flushing mob file 997d8d2a525d3ba7cd152ef78c94cfd4/A: appending metadata at 1732634757851 (+10 ms)Flushing mob file 997d8d2a525d3ba7cd152ef78c94cfd4/A: closing flushed file at 1732634757851Flushing 997d8d2a525d3ba7cd152ef78c94cfd4/A: appending metadata at 1732634758269 (+418 ms)Flushing 997d8d2a525d3ba7cd152ef78c94cfd4/A: closing flushed file at 1732634758269Flushing 997d8d2a525d3ba7cd152ef78c94cfd4/B: creating writer at 1732634758685 (+416 ms)Flushing 997d8d2a525d3ba7cd152ef78c94cfd4/B: appending metadata at 1732634758689 (+4 ms)Flushing 997d8d2a525d3ba7cd152ef78c94cfd4/B: closing flushed file at 1732634758689Flushing 997d8d2a525d3ba7cd152ef78c94cfd4/C: creating writer at 1732634759106 (+417 ms)Flushing 997d8d2a525d3ba7cd152ef78c94cfd4/C: appending metadata at 1732634759109 (+3 ms)Flushing 997d8d2a525d3ba7cd152ef78c94cfd4/C: closing flushed file at 1732634759110 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37fba929: reopening flushed file at 1732634759524 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b1bcb70: reopening flushed file at 1732634759529 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29381df5: reopening flushed file at 1732634759534 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 997d8d2a525d3ba7cd152ef78c94cfd4 in 1703ms, sequenceid=423, compaction requested=true at 1732634759541 (+7 ms)Writing region close event to WAL at 1732634759627 (+86 ms)Running coprocessor post-close hooks at 1732634759629 (+2 ms)Closed at 1732634759629 2024-11-26T15:25:59,631 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed 997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,631 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=997d8d2a525d3ba7cd152ef78c94cfd4, regionState=CLOSED 2024-11-26T15:25:59,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:25:59,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-26T15:25:59,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure 997d8d2a525d3ba7cd152ef78c94cfd4, server=6cc2bc8bf733,45729,1732634697146 in 1.9480 sec 2024-11-26T15:25:59,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-26T15:25:59,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=997d8d2a525d3ba7cd152ef78c94cfd4, UNASSIGN in 1.9530 sec 2024-11-26T15:25:59,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-26T15:25:59,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9560 sec 2024-11-26T15:25:59,641 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634759640"}]},"ts":"1732634759640"} 2024-11-26T15:25:59,642 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T15:25:59,642 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T15:25:59,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9710 sec 2024-11-26T15:25:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T15:25:59,824 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:59,826 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T15:25:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:59,831 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T15:25:59,833 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:59,836 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,840 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/recovered.edits] 2024-11-26T15:25:59,845 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/0087f912bc4b4a138aba13c3dcabf229 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/0087f912bc4b4a138aba13c3dcabf229 2024-11-26T15:25:59,847 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/11672751f0344fb9821bc28c13ba7e75 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/11672751f0344fb9821bc28c13ba7e75 2024-11-26T15:25:59,849 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/2170043f08274c75a3b4a459f2b0d7ad to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/2170043f08274c75a3b4a459f2b0d7ad 2024-11-26T15:25:59,851 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7d37d9be447b4994841ec00648684774 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/A/7d37d9be447b4994841ec00648684774 2024-11-26T15:25:59,855 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/85c3dca37f7747149aff45e450f9882f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/85c3dca37f7747149aff45e450f9882f 2024-11-26T15:25:59,857 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/bbf81d2779cb47bbb688cd6bbac654de to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/bbf81d2779cb47bbb688cd6bbac654de 2024-11-26T15:25:59,859 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/caf59a066e0045bcb8b0a0e5b9f79921 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/caf59a066e0045bcb8b0a0e5b9f79921 2024-11-26T15:25:59,861 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/ec76a0bba2c54c43bdf8d235ab5c4c6a to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/B/ec76a0bba2c54c43bdf8d235ab5c4c6a 2024-11-26T15:25:59,865 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/341a61b9a0be4a829df52e9ddfb19d1e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/341a61b9a0be4a829df52e9ddfb19d1e 2024-11-26T15:25:59,866 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/d811cd9e403e4259afa523e1c7afb101 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/d811cd9e403e4259afa523e1c7afb101 2024-11-26T15:25:59,867 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/ee9cff97dee749a8bf21e57255b0e394 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/ee9cff97dee749a8bf21e57255b0e394 2024-11-26T15:25:59,868 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/f3139d4e02fd4b93aede76f32ee6999a to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/C/f3139d4e02fd4b93aede76f32ee6999a 2024-11-26T15:25:59,870 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/recovered.edits/426.seqid to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4/recovered.edits/426.seqid 2024-11-26T15:25:59,871 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,871 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T15:25:59,871 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T15:25:59,872 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T15:25:59,875 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260085bfb6cb704622aece5b3daee8a077_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260085bfb6cb704622aece5b3daee8a077_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,876 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260aa07df34abd4cbe8d8b065d6a5d53ea_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260aa07df34abd4cbe8d8b065d6a5d53ea_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,877 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260d3a9fc2ba2e4c6f8c823ff9c8ac61f8_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260d3a9fc2ba2e4c6f8c823ff9c8ac61f8_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,878 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126286925bd15a5451eb34bbc05eaa069ba_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126286925bd15a5451eb34bbc05eaa069ba_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,879 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112629b3b97a5d9a472793f10be979ea80a5_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112629b3b97a5d9a472793f10be979ea80a5_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,880 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262f03fabe898744d19634fae94d7eabdf_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262f03fabe898744d19634fae94d7eabdf_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,881 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112635af25d66cb34c1c9e3cf27fce3b7a43_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112635af25d66cb34c1c9e3cf27fce3b7a43_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,882 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126395f202ebb3e4cf487bee2251e5dd6a9_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126395f202ebb3e4cf487bee2251e5dd6a9_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,883 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126414436872270432d880453d89694bad5_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126414436872270432d880453d89694bad5_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,884 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112653e7b6c5ee7d4707b001d11703475ee9_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112653e7b6c5ee7d4707b001d11703475ee9_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,884 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265a98b5000c334e8caf9e142cee837a08_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265a98b5000c334e8caf9e142cee837a08_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,885 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f1a1bec39294ffbbfdec58208f717c8_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f1a1bec39294ffbbfdec58208f717c8_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,886 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126777693ee3a7f4da2868a75169f1c9c66_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126777693ee3a7f4da2868a75169f1c9c66_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,887 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112687eab6d7f1ba472896b4e4020209f837_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112687eab6d7f1ba472896b4e4020209f837_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,888 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a8b2cbaede974f07a81cad6d877eaee2_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a8b2cbaede974f07a81cad6d877eaee2_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,889 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b488f85fe9b143a1bf52f439e51d64af_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b488f85fe9b143a1bf52f439e51d64af_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,890 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bc84e226e319464da219611412374e3c_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bc84e226e319464da219611412374e3c_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,891 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c092ef95f3774e09a0c2513abe0e7ee4_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c092ef95f3774e09a0c2513abe0e7ee4_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,892 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c6b73e16c0bd4e698dcb48d87384cfaa_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c6b73e16c0bd4e698dcb48d87384cfaa_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,893 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d660e34f41ca4bb096c93d549f7f8156_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d660e34f41ca4bb096c93d549f7f8156_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,894 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d930a408f3f246fba4d487e0eb25f233_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d930a408f3f246fba4d487e0eb25f233_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,895 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e8d7327a7f27485d9e86e7c5df4d55cd_997d8d2a525d3ba7cd152ef78c94cfd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e8d7327a7f27485d9e86e7c5df4d55cd_997d8d2a525d3ba7cd152ef78c94cfd4 2024-11-26T15:25:59,895 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T15:25:59,897 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:59,899 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T15:25:59,901 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T15:25:59,902 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:59,902 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T15:25:59,902 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732634759902"}]},"ts":"9223372036854775807"} 2024-11-26T15:25:59,904 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T15:25:59,904 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 997d8d2a525d3ba7cd152ef78c94cfd4, NAME => 'TestAcidGuarantees,,1732634728529.997d8d2a525d3ba7cd152ef78c94cfd4.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T15:25:59,904 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T15:25:59,904 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732634759904"}]},"ts":"9223372036854775807"} 2024-11-26T15:25:59,905 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T15:25:59,906 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:59,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 80 msec 2024-11-26T15:25:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T15:25:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T15:25:59,946 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:25:59,952 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 237) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-908112358_22 at /127.0.0.1:57604 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/cluster_1ef45550-199a-e5e8-727d-dd604c50436f/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/cluster_1ef45550-199a-e5e8-727d-dd604c50436f/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=590 (was 753), ProcessCount=11 (was 11), AvailableMemoryMB=9164 (was 9464) 2024-11-26T15:25:59,960 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=590, ProcessCount=11, AvailableMemoryMB=9163 2024-11-26T15:25:59,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T15:25:59,961 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T15:25:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T15:25:59,963 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T15:25:59,963 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:25:59,963 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-11-26T15:25:59,964 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T15:25:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-26T15:25:59,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742067_1243 (size=959) 2024-11-26T15:26:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-26T15:26:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-26T15:26:00,375 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6 2024-11-26T15:26:00,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742068_1244 (size=53) 2024-11-26T15:26:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-26T15:26:00,785 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:26:00,786 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 21fb246aea47afdec08e90541f5d61cc, disabling compactions & flushes 2024-11-26T15:26:00,786 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:00,786 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:00,786 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. after waiting 0 ms 2024-11-26T15:26:00,786 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:00,786 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:00,786 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 21fb246aea47afdec08e90541f5d61cc: Waiting for close lock at 1732634760785Disabling compacts and flushes for region at 1732634760785Disabling writes for close at 1732634760786 (+1 ms)Writing region close event to WAL at 1732634760786Closed at 1732634760786 2024-11-26T15:26:00,789 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T15:26:00,789 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732634760789"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732634760789"}]},"ts":"1732634760789"} 2024-11-26T15:26:00,793 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T15:26:00,795 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T15:26:00,796 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634760795"}]},"ts":"1732634760795"} 2024-11-26T15:26:00,799 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T15:26:00,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21fb246aea47afdec08e90541f5d61cc, ASSIGN}] 2024-11-26T15:26:00,801 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21fb246aea47afdec08e90541f5d61cc, ASSIGN 2024-11-26T15:26:00,801 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21fb246aea47afdec08e90541f5d61cc, ASSIGN; state=OFFLINE, location=6cc2bc8bf733,45729,1732634697146; forceNewPlan=false, retain=false 2024-11-26T15:26:00,952 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=21fb246aea47afdec08e90541f5d61cc, regionState=OPENING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:00,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21fb246aea47afdec08e90541f5d61cc, ASSIGN because future has completed 2024-11-26T15:26:00,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:26:01,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-26T15:26:01,114 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:01,114 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:26:01,115 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:01,115 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:26:01,115 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:01,115 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:01,117 INFO [StoreOpener-21fb246aea47afdec08e90541f5d61cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:01,118 INFO [StoreOpener-21fb246aea47afdec08e90541f5d61cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21fb246aea47afdec08e90541f5d61cc columnFamilyName A 2024-11-26T15:26:01,118 DEBUG [StoreOpener-21fb246aea47afdec08e90541f5d61cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:01,119 INFO [StoreOpener-21fb246aea47afdec08e90541f5d61cc-1 {}] regionserver.HStore(327): Store=21fb246aea47afdec08e90541f5d61cc/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:26:01,119 INFO [StoreOpener-21fb246aea47afdec08e90541f5d61cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:01,120 INFO [StoreOpener-21fb246aea47afdec08e90541f5d61cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21fb246aea47afdec08e90541f5d61cc columnFamilyName B 2024-11-26T15:26:01,120 DEBUG [StoreOpener-21fb246aea47afdec08e90541f5d61cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:01,121 INFO [StoreOpener-21fb246aea47afdec08e90541f5d61cc-1 {}] regionserver.HStore(327): Store=21fb246aea47afdec08e90541f5d61cc/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:26:01,121 INFO [StoreOpener-21fb246aea47afdec08e90541f5d61cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:01,122 INFO [StoreOpener-21fb246aea47afdec08e90541f5d61cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21fb246aea47afdec08e90541f5d61cc columnFamilyName C 2024-11-26T15:26:01,123 DEBUG [StoreOpener-21fb246aea47afdec08e90541f5d61cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:01,123 INFO [StoreOpener-21fb246aea47afdec08e90541f5d61cc-1 {}] regionserver.HStore(327): Store=21fb246aea47afdec08e90541f5d61cc/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:26:01,123 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:01,124 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:01,124 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:01,126 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:01,126 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:01,126 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T15:26:01,128 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:01,130 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:26:01,130 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened 21fb246aea47afdec08e90541f5d61cc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61842982, jitterRate=-0.07846775650978088}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T15:26:01,130 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:01,131 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for 21fb246aea47afdec08e90541f5d61cc: Running coprocessor pre-open hook at 1732634761115Writing region info on filesystem at 1732634761115Initializing all the Stores at 1732634761116 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634761116Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634761117 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634761117Cleaning up temporary data from old regions at 1732634761126 (+9 ms)Running coprocessor post-open hooks at 1732634761130 (+4 ms)Region opened successfully at 1732634761131 (+1 ms) 2024-11-26T15:26:01,131 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., pid=57, masterSystemTime=1732634761109 2024-11-26T15:26:01,133 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:01,133 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:01,133 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=21fb246aea47afdec08e90541f5d61cc, regionState=OPEN, openSeqNum=2, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:01,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:26:01,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-26T15:26:01,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure 21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 in 179 msec 2024-11-26T15:26:01,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-26T15:26:01,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21fb246aea47afdec08e90541f5d61cc, ASSIGN in 337 msec 2024-11-26T15:26:01,138 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T15:26:01,138 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634761138"}]},"ts":"1732634761138"} 2024-11-26T15:26:01,139 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T15:26:01,140 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T15:26:01,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-26T15:26:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-26T15:26:02,114 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:02,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24ada2c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,117 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:02,118 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:02,120 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2546dee7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:02,122 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:02,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eaa65f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,123 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:02,125 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:02,126 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T15:26:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 6cc2bc8bf733,37081,1732634696319 2024-11-26T15:26:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@68b416b 2024-11-26T15:26:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T15:26:02,128 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T15:26:02,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fbd4712, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,130 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:02,130 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:02,131 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:02,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7560e2a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:02,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:02,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:02,133 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:02,134 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@724e8cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6f1434, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,134 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:02,135 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:02,135 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79dfed85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:02,137 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:02,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53d2bfa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@634f1deb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,138 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:02,138 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:02,139 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:02,140 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2564c966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec2896e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:02,141 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:02,142 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:02,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:02,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f3824fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:02,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:02,144 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:02,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67abc23c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5155197b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,145 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:02,145 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:02,146 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:02,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:02,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:02,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b045a23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:02,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:02,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:02,147 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24167638, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2108806c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,148 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:02,148 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:02,149 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59253a03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:02,150 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:02,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b114dcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6935c50e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:02,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:02,151 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62c2e3e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:02,152 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:02,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4e66ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@767db965, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,153 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:02,154 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:02,154 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@817edbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:02,155 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5db6be22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6493a71c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:02,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:02,157 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:02,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:02,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:02,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a6b11b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:02,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:02,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:02,158 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:02,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46f4a651, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a1b1c7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:02,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:02,161 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3beef4f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:02,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:02,162 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:02,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f34337, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:02,163 DEBUG [Thread-1140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:02,163 DEBUG [Thread-1151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:02,163 DEBUG [Thread-1132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:02,164 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:02,164 DEBUG [Thread-1145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:02,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:02,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:02,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:02,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:02,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:02,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:02,165 DEBUG [Thread-1147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-26T15:26:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:02,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:02,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T15:26:02,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:02,166 DEBUG [Thread-1134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:02,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:02,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:02,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:02,167 DEBUG [Thread-1149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:02,167 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:02,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:26:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:26:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:26:02,168 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:02,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:02,168 DEBUG [Thread-1143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:26:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:26:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:02,171 DEBUG [Thread-1138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:02,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:02,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:02,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:02,172 DEBUG [Thread-1138 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:02,172 DEBUG [Thread-1136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:02,172 DEBUG [Thread-1138 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:26:02,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:02,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:26:02,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:02,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:02,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:26:02,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:26:02,176 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:26:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:02,180 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:02,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-26T15:26:02,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/fb5ecc43468e4529b3e07c8661672cd0 is 50, key is test_row_0/A:col10/1732634762178/Put/seqid=0 2024-11-26T15:26:02,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634822210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634822210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33556 deadline: 1732634822213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33610 deadline: 1732634822213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634822215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742069_1245 (size=14341) 2024-11-26T15:26:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T15:26:02,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T15:26:02,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:02,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:02,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:02,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:02,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:02,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634822322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634822322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634822322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33556 deadline: 1732634822322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33610 deadline: 1732634822324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T15:26:02,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:02,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:02,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:02,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:02,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:02,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:02,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T15:26:02,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634822532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634822532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33556 deadline: 1732634822532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33610 deadline: 1732634822533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634822532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T15:26:02,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:02,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:02,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:02,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:02,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:02,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/fb5ecc43468e4529b3e07c8661672cd0 2024-11-26T15:26:02,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/a51baa5b81a142f6ad75a2378272671b is 50, key is test_row_0/B:col10/1732634762178/Put/seqid=0 2024-11-26T15:26:02,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742070_1246 (size=12001) 2024-11-26T15:26:02,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T15:26:02,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:02,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:02,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:02,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:02,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:02,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T15:26:02,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33556 deadline: 1732634822842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634822842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33610 deadline: 1732634822842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634822842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634822842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:02,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:02,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:02,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T15:26:02,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:02,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:02,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:02,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:02,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:03,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/a51baa5b81a142f6ad75a2378272671b 2024-11-26T15:26:03,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T15:26:03,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:03,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:03,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:03,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:03,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/f9d27e5bab154c7aaca3f293a106560b is 50, key is test_row_0/C:col10/1732634762178/Put/seqid=0 2024-11-26T15:26:03,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742071_1247 (size=12001) 2024-11-26T15:26:03,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T15:26:03,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:03,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:03,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:03,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:03,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T15:26:03,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634823353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:03,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33556 deadline: 1732634823353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:03,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634823353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:03,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33610 deadline: 1732634823353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:03,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634823353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:03,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:03,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:03,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:03,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T15:26:03,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:03,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:03,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:03,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:03,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:03,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/f9d27e5bab154c7aaca3f293a106560b 2024-11-26T15:26:03,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/fb5ecc43468e4529b3e07c8661672cd0 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/fb5ecc43468e4529b3e07c8661672cd0 2024-11-26T15:26:03,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/fb5ecc43468e4529b3e07c8661672cd0, entries=200, sequenceid=14, filesize=14.0 K 2024-11-26T15:26:03,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/a51baa5b81a142f6ad75a2378272671b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/a51baa5b81a142f6ad75a2378272671b 2024-11-26T15:26:03,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/a51baa5b81a142f6ad75a2378272671b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T15:26:03,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/f9d27e5bab154c7aaca3f293a106560b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/f9d27e5bab154c7aaca3f293a106560b 2024-11-26T15:26:03,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/f9d27e5bab154c7aaca3f293a106560b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T15:26:03,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 21fb246aea47afdec08e90541f5d61cc in 1338ms, sequenceid=14, compaction requested=false 2024-11-26T15:26:03,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:03,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-26T15:26:03,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:03,560 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-26T15:26:03,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/7734086310bb458492801a833cd71a85 is 50, key is test_row_0/A:col10/1732634762213/Put/seqid=0 2024-11-26T15:26:03,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742072_1248 (size=12001) 2024-11-26T15:26:03,974 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/7734086310bb458492801a833cd71a85 2024-11-26T15:26:03,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/233b73eac05f49bca14e94f7a523feca is 50, key is test_row_0/B:col10/1732634762213/Put/seqid=0 2024-11-26T15:26:03,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742073_1249 (size=12001) 2024-11-26T15:26:04,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T15:26:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:04,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:04,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:04,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634824370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:04,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634824371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:04,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634824372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:04,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33556 deadline: 1732634824372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:04,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33610 deadline: 1732634824372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:04,375 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in 21fb246aea47afdec08e90541f5d61cc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:04,375 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in 21fb246aea47afdec08e90541f5d61cc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:04,396 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/233b73eac05f49bca14e94f7a523feca 2024-11-26T15:26:04,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/cab214d9b343481cb1e69ec7ac585fa7 is 50, key is test_row_0/C:col10/1732634762213/Put/seqid=0 2024-11-26T15:26:04,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742074_1250 (size=12001) 2024-11-26T15:26:04,410 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/cab214d9b343481cb1e69ec7ac585fa7 2024-11-26T15:26:04,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/7734086310bb458492801a833cd71a85 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/7734086310bb458492801a833cd71a85 2024-11-26T15:26:04,418 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/7734086310bb458492801a833cd71a85, entries=150, sequenceid=37, filesize=11.7 K 2024-11-26T15:26:04,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/233b73eac05f49bca14e94f7a523feca as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/233b73eac05f49bca14e94f7a523feca 2024-11-26T15:26:04,423 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/233b73eac05f49bca14e94f7a523feca, entries=150, sequenceid=37, filesize=11.7 K 2024-11-26T15:26:04,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/cab214d9b343481cb1e69ec7ac585fa7 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/cab214d9b343481cb1e69ec7ac585fa7 2024-11-26T15:26:04,430 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/cab214d9b343481cb1e69ec7ac585fa7, entries=150, sequenceid=37, filesize=11.7 K 2024-11-26T15:26:04,431 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 21fb246aea47afdec08e90541f5d61cc in 871ms, sequenceid=37, compaction requested=false 2024-11-26T15:26:04,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:04,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:04,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-26T15:26:04,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-26T15:26:04,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-26T15:26:04,438 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2640 sec 2024-11-26T15:26:04,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.2740 sec 2024-11-26T15:26:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:04,486 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:04,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-26T15:26:04,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/e823b65524bc4d4c9a2340a0a022c255 is 50, key is test_row_0/A:col10/1732634764371/Put/seqid=0 2024-11-26T15:26:04,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742075_1251 (size=14341) 2024-11-26T15:26:04,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634824517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:04,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634824518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:04,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:04,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:04,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634824522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:04,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:04,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634824633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:04,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634824636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634824636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:04,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:04,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:04,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634824842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:04,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634824843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:04,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:04,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634824843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:04,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:04,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:04,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:04,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/e823b65524bc4d4c9a2340a0a022c255 2024-11-26T15:26:04,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/86489ac4509542ea9775026aaae5f392 is 50, key is test_row_0/B:col10/1732634764371/Put/seqid=0 2024-11-26T15:26:04,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742076_1252 (size=12001) 2024-11-26T15:26:05,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634825153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:05,157 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T15:26:05,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:05,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634825159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634825159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:05,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/86489ac4509542ea9775026aaae5f392 2024-11-26T15:26:05,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/daa86c5deb014f3f91d2b090461e8451 is 50, key is test_row_0/C:col10/1732634764371/Put/seqid=0 2024-11-26T15:26:05,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742077_1253 (size=12001) 2024-11-26T15:26:05,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/daa86c5deb014f3f91d2b090461e8451 2024-11-26T15:26:05,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/e823b65524bc4d4c9a2340a0a022c255 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/e823b65524bc4d4c9a2340a0a022c255 2024-11-26T15:26:05,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/e823b65524bc4d4c9a2340a0a022c255, entries=200, sequenceid=53, filesize=14.0 K 2024-11-26T15:26:05,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/86489ac4509542ea9775026aaae5f392 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/86489ac4509542ea9775026aaae5f392 2024-11-26T15:26:05,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/86489ac4509542ea9775026aaae5f392, entries=150, sequenceid=53, filesize=11.7 K 2024-11-26T15:26:05,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/daa86c5deb014f3f91d2b090461e8451 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/daa86c5deb014f3f91d2b090461e8451 2024-11-26T15:26:05,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/daa86c5deb014f3f91d2b090461e8451, entries=150, sequenceid=53, filesize=11.7 K 2024-11-26T15:26:05,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 21fb246aea47afdec08e90541f5d61cc in 916ms, sequenceid=53, compaction requested=true 2024-11-26T15:26:05,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:05,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:26:05,402 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:05,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:05,402 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:05,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:26:05,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:05,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:26:05,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:05,403 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:05,403 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:05,403 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/A is initiating minor compaction (all files) 2024-11-26T15:26:05,403 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/B is initiating minor compaction (all files) 2024-11-26T15:26:05,403 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/A in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:05,403 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/B in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:05,403 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/a51baa5b81a142f6ad75a2378272671b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/233b73eac05f49bca14e94f7a523feca, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/86489ac4509542ea9775026aaae5f392] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=35.2 K 2024-11-26T15:26:05,403 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/fb5ecc43468e4529b3e07c8661672cd0, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/7734086310bb458492801a833cd71a85, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/e823b65524bc4d4c9a2340a0a022c255] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=39.7 K 2024-11-26T15:26:05,405 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb5ecc43468e4529b3e07c8661672cd0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732634762178 2024-11-26T15:26:05,405 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting a51baa5b81a142f6ad75a2378272671b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732634762178 2024-11-26T15:26:05,405 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7734086310bb458492801a833cd71a85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732634762207 2024-11-26T15:26:05,405 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 233b73eac05f49bca14e94f7a523feca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732634762207 2024-11-26T15:26:05,405 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting e823b65524bc4d4c9a2340a0a022c255, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732634764371 2024-11-26T15:26:05,405 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 86489ac4509542ea9775026aaae5f392, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732634764371 2024-11-26T15:26:05,426 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#A#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:05,430 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/eb1445d68e604d08a6d050a32544d509 is 50, key is test_row_0/A:col10/1732634764371/Put/seqid=0 2024-11-26T15:26:05,442 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#B#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:05,445 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/3b0f8b5bec654f0197675d2715e7d315 is 50, key is test_row_0/B:col10/1732634764371/Put/seqid=0 2024-11-26T15:26:05,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742078_1254 (size=12104) 2024-11-26T15:26:05,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742079_1255 (size=12104) 2024-11-26T15:26:05,486 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/eb1445d68e604d08a6d050a32544d509 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/eb1445d68e604d08a6d050a32544d509 2024-11-26T15:26:05,496 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/A of 21fb246aea47afdec08e90541f5d61cc into eb1445d68e604d08a6d050a32544d509(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:05,496 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:05,496 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/A, priority=13, startTime=1732634765402; duration=0sec 2024-11-26T15:26:05,496 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:05,496 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:A 2024-11-26T15:26:05,497 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:05,500 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:05,500 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/C is initiating minor compaction (all files) 2024-11-26T15:26:05,500 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/C in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:05,500 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/f9d27e5bab154c7aaca3f293a106560b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/cab214d9b343481cb1e69ec7ac585fa7, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/daa86c5deb014f3f91d2b090461e8451] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=35.2 K 2024-11-26T15:26:05,505 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9d27e5bab154c7aaca3f293a106560b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732634762178 2024-11-26T15:26:05,505 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/3b0f8b5bec654f0197675d2715e7d315 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/3b0f8b5bec654f0197675d2715e7d315 2024-11-26T15:26:05,505 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting cab214d9b343481cb1e69ec7ac585fa7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732634762207 2024-11-26T15:26:05,506 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting daa86c5deb014f3f91d2b090461e8451, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732634764371 2024-11-26T15:26:05,519 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/B of 21fb246aea47afdec08e90541f5d61cc into 3b0f8b5bec654f0197675d2715e7d315(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:05,519 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:05,519 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/B, priority=13, startTime=1732634765402; duration=0sec 2024-11-26T15:26:05,519 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:05,519 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:B 2024-11-26T15:26:05,522 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#C#compaction#208 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:05,523 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/47d61a2f52ef4a3e8001890388dfa0b4 is 50, key is test_row_0/C:col10/1732634764371/Put/seqid=0 2024-11-26T15:26:05,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742080_1256 (size=12104) 2024-11-26T15:26:05,552 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/47d61a2f52ef4a3e8001890388dfa0b4 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/47d61a2f52ef4a3e8001890388dfa0b4 2024-11-26T15:26:05,561 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/C of 21fb246aea47afdec08e90541f5d61cc into 47d61a2f52ef4a3e8001890388dfa0b4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:05,561 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:05,561 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/C, priority=13, startTime=1732634765402; duration=0sec 2024-11-26T15:26:05,561 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:05,561 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:C 2024-11-26T15:26:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:05,665 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:05,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-26T15:26:05,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/bb970c5d57a84a6cb672d71729041891 is 50, key is test_row_0/A:col10/1732634764520/Put/seqid=0 2024-11-26T15:26:05,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634825694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:05,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634825695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:05,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742081_1257 (size=14341) 2024-11-26T15:26:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:05,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634825698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:05,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/bb970c5d57a84a6cb672d71729041891 2024-11-26T15:26:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:05,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/a5b19dd2ca314b18abdc360ecfdc9e41 is 50, key is test_row_0/B:col10/1732634764520/Put/seqid=0 2024-11-26T15:26:05,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742082_1258 (size=12001) 2024-11-26T15:26:05,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634825803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:05,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634825812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:05,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634825812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:05,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:06,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634826013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:06,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634826022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:06,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634826022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:06,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/a5b19dd2ca314b18abdc360ecfdc9e41 2024-11-26T15:26:06,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/0432fcd906f64a4bb08e8dfee5b6aa74 is 50, key is test_row_0/C:col10/1732634764520/Put/seqid=0 2024-11-26T15:26:06,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742083_1259 (size=12001) 2024-11-26T15:26:06,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634826323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-26T15:26:06,332 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:06,333 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-26T15:26:06,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634826332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T15:26:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:06,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634826332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:06,334 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:06,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:06,335 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:06,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:06,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33556 deadline: 1732634826393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:06,395 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in 21fb246aea47afdec08e90541f5d61cc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:06,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33610 deadline: 1732634826402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:06,404 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in 21fb246aea47afdec08e90541f5d61cc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:06,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:06,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:06,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T15:26:06,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T15:26:06,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:06,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:06,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:06,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:06,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:06,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/0432fcd906f64a4bb08e8dfee5b6aa74 2024-11-26T15:26:06,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/bb970c5d57a84a6cb672d71729041891 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/bb970c5d57a84a6cb672d71729041891 2024-11-26T15:26:06,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/bb970c5d57a84a6cb672d71729041891, entries=200, sequenceid=78, filesize=14.0 K 2024-11-26T15:26:06,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/a5b19dd2ca314b18abdc360ecfdc9e41 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/a5b19dd2ca314b18abdc360ecfdc9e41 2024-11-26T15:26:06,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/a5b19dd2ca314b18abdc360ecfdc9e41, entries=150, sequenceid=78, filesize=11.7 K 2024-11-26T15:26:06,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/0432fcd906f64a4bb08e8dfee5b6aa74 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/0432fcd906f64a4bb08e8dfee5b6aa74 2024-11-26T15:26:06,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/0432fcd906f64a4bb08e8dfee5b6aa74, entries=150, sequenceid=78, filesize=11.7 K 2024-11-26T15:26:06,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=87.22 KB/89310 for 21fb246aea47afdec08e90541f5d61cc in 928ms, sequenceid=78, compaction requested=false 2024-11-26T15:26:06,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:06,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-26T15:26:06,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:06,643 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-26T15:26:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T15:26:06,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/a9d89a42cdeb46ccb87ec8340baef9c1 is 50, key is test_row_0/A:col10/1732634765691/Put/seqid=0 2024-11-26T15:26:06,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742084_1260 (size=12001) 2024-11-26T15:26:06,706 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/a9d89a42cdeb46ccb87ec8340baef9c1 2024-11-26T15:26:06,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/c6585a62282945af93b9f852c106330c is 50, key is test_row_0/B:col10/1732634765691/Put/seqid=0 2024-11-26T15:26:06,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742085_1261 (size=12001) 2024-11-26T15:26:06,764 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/c6585a62282945af93b9f852c106330c 2024-11-26T15:26:06,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/4577f58b0eed4c9cbe63c11a184433fc is 50, key is test_row_0/C:col10/1732634765691/Put/seqid=0 2024-11-26T15:26:06,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742086_1262 (size=12001) 2024-11-26T15:26:06,812 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/4577f58b0eed4c9cbe63c11a184433fc 2024-11-26T15:26:06,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/a9d89a42cdeb46ccb87ec8340baef9c1 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a9d89a42cdeb46ccb87ec8340baef9c1 2024-11-26T15:26:06,829 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a9d89a42cdeb46ccb87ec8340baef9c1, entries=150, sequenceid=94, filesize=11.7 K 2024-11-26T15:26:06,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/c6585a62282945af93b9f852c106330c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/c6585a62282945af93b9f852c106330c 2024-11-26T15:26:06,838 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/c6585a62282945af93b9f852c106330c, entries=150, sequenceid=94, filesize=11.7 K 2024-11-26T15:26:06,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/4577f58b0eed4c9cbe63c11a184433fc as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/4577f58b0eed4c9cbe63c11a184433fc 2024-11-26T15:26:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:06,846 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/4577f58b0eed4c9cbe63c11a184433fc, entries=150, sequenceid=94, filesize=11.7 K 2024-11-26T15:26:06,847 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=6.71 KB/6870 for 21fb246aea47afdec08e90541f5d61cc in 204ms, sequenceid=94, compaction requested=true 2024-11-26T15:26:06,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:06,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:06,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-26T15:26:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-26T15:26:06,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-26T15:26:06,855 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 515 msec 2024-11-26T15:26:06,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 522 msec 2024-11-26T15:26:06,896 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:06,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T15:26:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:06,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/f0630962497a49f48f10caad7febc5a7 is 50, key is test_row_0/A:col10/1732634766876/Put/seqid=0 2024-11-26T15:26:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T15:26:06,963 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:06,964 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:06,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742087_1263 (size=14341) 2024-11-26T15:26:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-26T15:26:06,966 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T15:26:06,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/f0630962497a49f48f10caad7febc5a7 2024-11-26T15:26:06,967 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:06,967 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:06,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/0c62ae1f11854adda942eba096a84392 is 50, key is test_row_0/B:col10/1732634766876/Put/seqid=0 2024-11-26T15:26:07,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634827003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634827010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634827013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742088_1264 (size=12001) 2024-11-26T15:26:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T15:26:07,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T15:26:07,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:07,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:07,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:07,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:07,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:07,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634827123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634827123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634827133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T15:26:07,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:07,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:07,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:07,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:07,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:07,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T15:26:07,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634827332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634827343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634827343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T15:26:07,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:07,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:07,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:07,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:07,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:07,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/0c62ae1f11854adda942eba096a84392 2024-11-26T15:26:07,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/bbbb25bda2ab4a139bfd45f5f89d57c5 is 50, key is test_row_0/C:col10/1732634766876/Put/seqid=0 2024-11-26T15:26:07,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742089_1265 (size=12001) 2024-11-26T15:26:07,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/bbbb25bda2ab4a139bfd45f5f89d57c5 2024-11-26T15:26:07,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/f0630962497a49f48f10caad7febc5a7 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/f0630962497a49f48f10caad7febc5a7 2024-11-26T15:26:07,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/f0630962497a49f48f10caad7febc5a7, entries=200, sequenceid=105, filesize=14.0 K 2024-11-26T15:26:07,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/0c62ae1f11854adda942eba096a84392 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0c62ae1f11854adda942eba096a84392 2024-11-26T15:26:07,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0c62ae1f11854adda942eba096a84392, entries=150, sequenceid=105, filesize=11.7 K 2024-11-26T15:26:07,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/bbbb25bda2ab4a139bfd45f5f89d57c5 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/bbbb25bda2ab4a139bfd45f5f89d57c5 2024-11-26T15:26:07,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/bbbb25bda2ab4a139bfd45f5f89d57c5, entries=150, sequenceid=105, filesize=11.7 K 2024-11-26T15:26:07,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 21fb246aea47afdec08e90541f5d61cc in 622ms, sequenceid=105, compaction requested=true 2024-11-26T15:26:07,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:07,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:26:07,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:07,519 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:26:07,519 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:26:07,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:26:07,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:07,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:26:07,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:07,521 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:26:07,521 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/A is initiating minor compaction (all files) 2024-11-26T15:26:07,521 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/A in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:07,521 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/eb1445d68e604d08a6d050a32544d509, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/bb970c5d57a84a6cb672d71729041891, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a9d89a42cdeb46ccb87ec8340baef9c1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/f0630962497a49f48f10caad7febc5a7] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=51.5 K 2024-11-26T15:26:07,521 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:26:07,521 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/B is initiating minor compaction (all files) 2024-11-26T15:26:07,521 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/B in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:07,521 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/3b0f8b5bec654f0197675d2715e7d315, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/a5b19dd2ca314b18abdc360ecfdc9e41, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/c6585a62282945af93b9f852c106330c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0c62ae1f11854adda942eba096a84392] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=47.0 K 2024-11-26T15:26:07,522 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb1445d68e604d08a6d050a32544d509, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732634764371 2024-11-26T15:26:07,523 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b0f8b5bec654f0197675d2715e7d315, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732634764371 2024-11-26T15:26:07,523 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting a5b19dd2ca314b18abdc360ecfdc9e41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732634764517 2024-11-26T15:26:07,523 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb970c5d57a84a6cb672d71729041891, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732634764517 2024-11-26T15:26:07,524 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting c6585a62282945af93b9f852c106330c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732634765691 2024-11-26T15:26:07,524 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9d89a42cdeb46ccb87ec8340baef9c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732634765691 2024-11-26T15:26:07,524 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c62ae1f11854adda942eba096a84392, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732634766876 2024-11-26T15:26:07,524 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0630962497a49f48f10caad7febc5a7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732634766876 2024-11-26T15:26:07,539 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#B#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:07,539 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/5ef107c85ad34bbf92fa8078901d2373 is 50, key is test_row_0/B:col10/1732634766876/Put/seqid=0 2024-11-26T15:26:07,549 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#A#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:07,550 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/4820f58262c445fa8c1190f92c0b650e is 50, key is test_row_0/A:col10/1732634766876/Put/seqid=0 2024-11-26T15:26:07,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742090_1266 (size=12241) 2024-11-26T15:26:07,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742091_1267 (size=12241) 2024-11-26T15:26:07,574 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/4820f58262c445fa8c1190f92c0b650e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/4820f58262c445fa8c1190f92c0b650e 2024-11-26T15:26:07,580 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/A of 21fb246aea47afdec08e90541f5d61cc into 4820f58262c445fa8c1190f92c0b650e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:07,580 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:07,580 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/A, priority=12, startTime=1732634767519; duration=0sec 2024-11-26T15:26:07,580 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:07,580 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:A 2024-11-26T15:26:07,580 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:26:07,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-26T15:26:07,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:07,582 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-26T15:26:07,583 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:26:07,583 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/C is initiating minor compaction (all files) 2024-11-26T15:26:07,583 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/C in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:07,583 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/47d61a2f52ef4a3e8001890388dfa0b4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/0432fcd906f64a4bb08e8dfee5b6aa74, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/4577f58b0eed4c9cbe63c11a184433fc, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/bbbb25bda2ab4a139bfd45f5f89d57c5] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=47.0 K 2024-11-26T15:26:07,583 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47d61a2f52ef4a3e8001890388dfa0b4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732634764371 2024-11-26T15:26:07,583 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0432fcd906f64a4bb08e8dfee5b6aa74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732634764517 2024-11-26T15:26:07,584 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4577f58b0eed4c9cbe63c11a184433fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732634765691 2024-11-26T15:26:07,585 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbbb25bda2ab4a139bfd45f5f89d57c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732634766876 2024-11-26T15:26:07,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/ed83b7ef7e01454a91dd3aa88292572f is 50, key is test_row_0/A:col10/1732634766999/Put/seqid=0 2024-11-26T15:26:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T15:26:07,603 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:07,603 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/ebe9cef98bc14eee9118dcee66b38a42 is 50, key is test_row_0/C:col10/1732634766876/Put/seqid=0 2024-11-26T15:26:07,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742092_1268 (size=12051) 2024-11-26T15:26:07,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742093_1269 (size=12241) 2024-11-26T15:26:07,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:07,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:07,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634827655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634827657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634827659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634827762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634827772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634827772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,961 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/5ef107c85ad34bbf92fa8078901d2373 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/5ef107c85ad34bbf92fa8078901d2373 2024-11-26T15:26:07,966 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/B of 21fb246aea47afdec08e90541f5d61cc into 5ef107c85ad34bbf92fa8078901d2373(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:07,966 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:07,966 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/B, priority=12, startTime=1732634767519; duration=0sec 2024-11-26T15:26:07,966 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:07,967 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:B 2024-11-26T15:26:07,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634827972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634827982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:07,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634827982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:07,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:07,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:07,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:08,015 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/ed83b7ef7e01454a91dd3aa88292572f 2024-11-26T15:26:08,020 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/ebe9cef98bc14eee9118dcee66b38a42 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/ebe9cef98bc14eee9118dcee66b38a42 2024-11-26T15:26:08,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/fc29ea180e1f4a4ab6173ee9793176c8 is 50, key is test_row_0/B:col10/1732634766999/Put/seqid=0 2024-11-26T15:26:08,027 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/C of 21fb246aea47afdec08e90541f5d61cc into ebe9cef98bc14eee9118dcee66b38a42(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:08,027 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:08,027 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/C, priority=12, startTime=1732634767520; duration=0sec 2024-11-26T15:26:08,027 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:08,027 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:C 2024-11-26T15:26:08,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742094_1270 (size=12051) 2024-11-26T15:26:08,033 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/fc29ea180e1f4a4ab6173ee9793176c8 2024-11-26T15:26:08,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/688bdbcf1a274204aafd6f539de3c168 is 50, key is test_row_0/C:col10/1732634766999/Put/seqid=0 2024-11-26T15:26:08,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742095_1271 (size=12051) 2024-11-26T15:26:08,055 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/688bdbcf1a274204aafd6f539de3c168 2024-11-26T15:26:08,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/ed83b7ef7e01454a91dd3aa88292572f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/ed83b7ef7e01454a91dd3aa88292572f 2024-11-26T15:26:08,065 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/ed83b7ef7e01454a91dd3aa88292572f, entries=150, sequenceid=131, filesize=11.8 K 2024-11-26T15:26:08,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/fc29ea180e1f4a4ab6173ee9793176c8 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/fc29ea180e1f4a4ab6173ee9793176c8 2024-11-26T15:26:08,071 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/fc29ea180e1f4a4ab6173ee9793176c8, entries=150, sequenceid=131, filesize=11.8 K 2024-11-26T15:26:08,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/688bdbcf1a274204aafd6f539de3c168 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/688bdbcf1a274204aafd6f539de3c168 2024-11-26T15:26:08,077 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/688bdbcf1a274204aafd6f539de3c168, entries=150, sequenceid=131, filesize=11.8 K 2024-11-26T15:26:08,078 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 21fb246aea47afdec08e90541f5d61cc in 496ms, sequenceid=131, compaction requested=false 2024-11-26T15:26:08,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:08,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:08,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-26T15:26:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-26T15:26:08,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-26T15:26:08,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1130 sec 2024-11-26T15:26:08,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.1180 sec 2024-11-26T15:26:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-26T15:26:08,102 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:08,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-26T15:26:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T15:26:08,104 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:08,105 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:08,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T15:26:08,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T15:26:08,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:08,258 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T15:26:08,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/5aca6befd0a84d3d8f2015a01b2d1639 is 50, key is test_row_0/A:col10/1732634767656/Put/seqid=0 2024-11-26T15:26:08,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742096_1272 (size=12151) 2024-11-26T15:26:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:08,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:08,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634828319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:08,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634828320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:08,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634828321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T15:26:08,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634828432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634828432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:08,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634828432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:08,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:08,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634828642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:08,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634828642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:08,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634828642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:08,668 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/5aca6befd0a84d3d8f2015a01b2d1639 2024-11-26T15:26:08,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/49d1cb55ca634bd98e7891d687420a20 is 50, key is test_row_0/B:col10/1732634767656/Put/seqid=0 2024-11-26T15:26:08,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742097_1273 (size=12151) 2024-11-26T15:26:08,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T15:26:08,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634828953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:08,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634828953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:08,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634828953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:09,081 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/49d1cb55ca634bd98e7891d687420a20 2024-11-26T15:26:09,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/9691a598f17d4c76841e14b0684eebfd is 50, key is test_row_0/C:col10/1732634767656/Put/seqid=0 2024-11-26T15:26:09,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742098_1274 (size=12151) 2024-11-26T15:26:09,107 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/9691a598f17d4c76841e14b0684eebfd 2024-11-26T15:26:09,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/5aca6befd0a84d3d8f2015a01b2d1639 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/5aca6befd0a84d3d8f2015a01b2d1639 2024-11-26T15:26:09,118 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/5aca6befd0a84d3d8f2015a01b2d1639, entries=150, sequenceid=144, filesize=11.9 K 2024-11-26T15:26:09,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/49d1cb55ca634bd98e7891d687420a20 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/49d1cb55ca634bd98e7891d687420a20 2024-11-26T15:26:09,125 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/49d1cb55ca634bd98e7891d687420a20, entries=150, sequenceid=144, filesize=11.9 K 2024-11-26T15:26:09,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/9691a598f17d4c76841e14b0684eebfd as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/9691a598f17d4c76841e14b0684eebfd 2024-11-26T15:26:09,133 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/9691a598f17d4c76841e14b0684eebfd, entries=150, sequenceid=144, filesize=11.9 K 2024-11-26T15:26:09,133 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 21fb246aea47afdec08e90541f5d61cc in 875ms, sequenceid=144, compaction requested=true 2024-11-26T15:26:09,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:09,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:09,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-26T15:26:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-26T15:26:09,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-26T15:26:09,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0310 sec 2024-11-26T15:26:09,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.0360 sec 2024-11-26T15:26:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T15:26:09,243 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:09,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-26T15:26:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T15:26:09,245 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:09,246 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:09,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T15:26:09,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T15:26:09,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:09,398 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-26T15:26:09,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/cf29669c2f314b9c9c254535fd75e88f is 50, key is test_row_0/A:col10/1732634768314/Put/seqid=0 2024-11-26T15:26:09,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742099_1275 (size=12151) 2024-11-26T15:26:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:09,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:09,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634829471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:09,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634829474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:09,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634829475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T15:26:09,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634829583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:09,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:09,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634829583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:09,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:09,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:09,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634829583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:09,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:09,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:09,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:09,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634829792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634829792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634829792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:09,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:09,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:09,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:09,807 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/cf29669c2f314b9c9c254535fd75e88f 2024-11-26T15:26:09,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/33c07988b7ce4f2f88bcc04a58a45841 is 50, key is test_row_0/B:col10/1732634768314/Put/seqid=0 2024-11-26T15:26:09,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742100_1276 (size=12151) 2024-11-26T15:26:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T15:26:10,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634830103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:10,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:10,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634830103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634830103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:10,231 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/33c07988b7ce4f2f88bcc04a58a45841 2024-11-26T15:26:10,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/39b7d4f653504bac9f680f72bb058a99 is 50, key is test_row_0/C:col10/1732634768314/Put/seqid=0 2024-11-26T15:26:10,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742101_1277 (size=12151) 2024-11-26T15:26:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T15:26:10,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33610 deadline: 1732634830422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:10,425 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in 21fb246aea47afdec08e90541f5d61cc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:10,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:10,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33556 deadline: 1732634830443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:10,447 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in 21fb246aea47afdec08e90541f5d61cc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:10,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634830612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:10,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634830612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:10,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634830612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:10,642 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/39b7d4f653504bac9f680f72bb058a99 2024-11-26T15:26:10,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/cf29669c2f314b9c9c254535fd75e88f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/cf29669c2f314b9c9c254535fd75e88f 2024-11-26T15:26:10,650 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/cf29669c2f314b9c9c254535fd75e88f, entries=150, sequenceid=169, filesize=11.9 K 2024-11-26T15:26:10,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/33c07988b7ce4f2f88bcc04a58a45841 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/33c07988b7ce4f2f88bcc04a58a45841 2024-11-26T15:26:10,655 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/33c07988b7ce4f2f88bcc04a58a45841, entries=150, sequenceid=169, filesize=11.9 K 2024-11-26T15:26:10,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/39b7d4f653504bac9f680f72bb058a99 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/39b7d4f653504bac9f680f72bb058a99 2024-11-26T15:26:10,662 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/39b7d4f653504bac9f680f72bb058a99, entries=150, sequenceid=169, filesize=11.9 K 2024-11-26T15:26:10,663 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 21fb246aea47afdec08e90541f5d61cc in 1265ms, sequenceid=169, compaction requested=true 2024-11-26T15:26:10,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:10,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:10,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-26T15:26:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-26T15:26:10,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-26T15:26:10,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4200 sec 2024-11-26T15:26:10,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.4300 sec 2024-11-26T15:26:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T15:26:11,403 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:11,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-26T15:26:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T15:26:11,406 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:11,407 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:11,407 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T15:26:11,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T15:26:11,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:11,561 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T15:26:11,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/6b2c645068ef434fb52c8b8940da79ba is 50, key is test_row_0/A:col10/1732634769469/Put/seqid=0 2024-11-26T15:26:11,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742102_1278 (size=12151) 2024-11-26T15:26:11,619 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/6b2c645068ef434fb52c8b8940da79ba 2024-11-26T15:26:11,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:11,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:11,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/740104c616974171ac3fdd126d1b67af is 50, key is test_row_0/B:col10/1732634769469/Put/seqid=0 2024-11-26T15:26:11,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742103_1279 (size=12151) 2024-11-26T15:26:11,671 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/740104c616974171ac3fdd126d1b67af 2024-11-26T15:26:11,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634831678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:11,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634831679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:11,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:11,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:11,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634831685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:11,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:11,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/95743450405e4b0987797d70e395d6cf is 50, key is test_row_0/C:col10/1732634769469/Put/seqid=0 2024-11-26T15:26:11,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T15:26:11,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742104_1280 (size=12151) 2024-11-26T15:26:11,726 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/95743450405e4b0987797d70e395d6cf 2024-11-26T15:26:11,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/6b2c645068ef434fb52c8b8940da79ba as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/6b2c645068ef434fb52c8b8940da79ba 2024-11-26T15:26:11,751 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/6b2c645068ef434fb52c8b8940da79ba, entries=150, sequenceid=180, filesize=11.9 K 2024-11-26T15:26:11,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/740104c616974171ac3fdd126d1b67af as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/740104c616974171ac3fdd126d1b67af 2024-11-26T15:26:11,757 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/740104c616974171ac3fdd126d1b67af, entries=150, sequenceid=180, filesize=11.9 K 2024-11-26T15:26:11,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/95743450405e4b0987797d70e395d6cf as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/95743450405e4b0987797d70e395d6cf 2024-11-26T15:26:11,762 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/95743450405e4b0987797d70e395d6cf, entries=150, sequenceid=180, filesize=11.9 K 2024-11-26T15:26:11,763 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 21fb246aea47afdec08e90541f5d61cc in 202ms, sequenceid=180, compaction requested=true 2024-11-26T15:26:11,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:11,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:11,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-26T15:26:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-26T15:26:11,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-26T15:26:11,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 359 msec 2024-11-26T15:26:11,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 366 msec 2024-11-26T15:26:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:11,796 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:11,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-26T15:26:11,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/2e1faaff441040fd860a62a9ab8e1331 is 50, key is test_row_0/A:col10/1732634771676/Put/seqid=0 2024-11-26T15:26:11,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634831812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:11,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:11,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634831816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634831817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:11,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:11,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:11,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742105_1281 (size=14541) 2024-11-26T15:26:11,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634831932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:11,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:11,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634831932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:11,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:11,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634831932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T15:26:12,032 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:12,033 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-26T15:26:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T15:26:12,035 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:12,036 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:12,036 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T15:26:12,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634832142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634832142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:12,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634832142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:12,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T15:26:12,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:12,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:12,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:12,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/2e1faaff441040fd860a62a9ab8e1331 2024-11-26T15:26:12,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/ec495b19db85421f9ae9701dc627592e is 50, key is test_row_0/B:col10/1732634771676/Put/seqid=0 2024-11-26T15:26:12,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742106_1282 (size=12151) 2024-11-26T15:26:12,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T15:26:12,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:12,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:12,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:12,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T15:26:12,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634832452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:12,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634832453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:12,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634832463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:12,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T15:26:12,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:12,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:12,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:12,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T15:26:12,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:12,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:12,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:12,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T15:26:12,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/ec495b19db85421f9ae9701dc627592e 2024-11-26T15:26:12,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/072d1c41c6be43db81678189dd004054 is 50, key is test_row_0/C:col10/1732634771676/Put/seqid=0 2024-11-26T15:26:12,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742107_1283 (size=12151) 2024-11-26T15:26:12,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T15:26:12,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:12,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:12,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:12,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T15:26:12,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:12,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:12,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:12,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:12,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:12,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:12,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634832963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:12,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634832963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:12,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:12,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:12,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:12,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634832972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:12,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:12,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:13,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T15:26:13,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:13,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:13,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:13,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:13,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:13,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:13,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/072d1c41c6be43db81678189dd004054 2024-11-26T15:26:13,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/2e1faaff441040fd860a62a9ab8e1331 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/2e1faaff441040fd860a62a9ab8e1331 2024-11-26T15:26:13,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/2e1faaff441040fd860a62a9ab8e1331, entries=200, sequenceid=206, filesize=14.2 K 2024-11-26T15:26:13,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/ec495b19db85421f9ae9701dc627592e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/ec495b19db85421f9ae9701dc627592e 2024-11-26T15:26:13,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/ec495b19db85421f9ae9701dc627592e, entries=150, sequenceid=206, filesize=11.9 K 2024-11-26T15:26:13,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/072d1c41c6be43db81678189dd004054 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/072d1c41c6be43db81678189dd004054 2024-11-26T15:26:13,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/072d1c41c6be43db81678189dd004054, entries=150, sequenceid=206, filesize=11.9 K 2024-11-26T15:26:13,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 21fb246aea47afdec08e90541f5d61cc in 1339ms, sequenceid=206, compaction requested=true 2024-11-26T15:26:13,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:13,135 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-26T15:26:13,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:26:13,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:13,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:26:13,135 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-26T15:26:13,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:13,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:26:13,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:13,137 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 75286 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-26T15:26:13,137 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72896 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-26T15:26:13,137 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/B is initiating minor compaction (all files) 2024-11-26T15:26:13,137 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/A is initiating minor compaction (all files) 2024-11-26T15:26:13,137 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/A in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:13,137 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/4820f58262c445fa8c1190f92c0b650e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/ed83b7ef7e01454a91dd3aa88292572f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/5aca6befd0a84d3d8f2015a01b2d1639, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/cf29669c2f314b9c9c254535fd75e88f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/6b2c645068ef434fb52c8b8940da79ba, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/2e1faaff441040fd860a62a9ab8e1331] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=73.5 K 2024-11-26T15:26:13,137 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/B in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:13,137 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/5ef107c85ad34bbf92fa8078901d2373, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/fc29ea180e1f4a4ab6173ee9793176c8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/49d1cb55ca634bd98e7891d687420a20, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/33c07988b7ce4f2f88bcc04a58a45841, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/740104c616974171ac3fdd126d1b67af, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/ec495b19db85421f9ae9701dc627592e] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=71.2 K 2024-11-26T15:26:13,138 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4820f58262c445fa8c1190f92c0b650e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732634766876 2024-11-26T15:26:13,138 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ef107c85ad34bbf92fa8078901d2373, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732634766876 2024-11-26T15:26:13,138 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed83b7ef7e01454a91dd3aa88292572f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732634766999 2024-11-26T15:26:13,138 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting fc29ea180e1f4a4ab6173ee9793176c8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732634766999 2024-11-26T15:26:13,138 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 49d1cb55ca634bd98e7891d687420a20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732634767654 2024-11-26T15:26:13,139 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5aca6befd0a84d3d8f2015a01b2d1639, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732634767654 2024-11-26T15:26:13,139 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 33c07988b7ce4f2f88bcc04a58a45841, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732634768314 2024-11-26T15:26:13,139 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf29669c2f314b9c9c254535fd75e88f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732634768314 2024-11-26T15:26:13,139 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 740104c616974171ac3fdd126d1b67af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732634769469 2024-11-26T15:26:13,139 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b2c645068ef434fb52c8b8940da79ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732634769469 2024-11-26T15:26:13,139 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting ec495b19db85421f9ae9701dc627592e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732634771676 2024-11-26T15:26:13,140 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e1faaff441040fd860a62a9ab8e1331, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732634771674 2024-11-26T15:26:13,153 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#A#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:13,154 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/a45d857666e44854ab97afd8f49b2a31 is 50, key is test_row_0/A:col10/1732634771676/Put/seqid=0 2024-11-26T15:26:13,154 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#B#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:13,155 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/3a5d2502359a47d7a8537b4b7fd2083a is 50, key is test_row_0/B:col10/1732634771676/Put/seqid=0 2024-11-26T15:26:13,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742109_1285 (size=12595) 2024-11-26T15:26:13,169 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/a45d857666e44854ab97afd8f49b2a31 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a45d857666e44854ab97afd8f49b2a31 2024-11-26T15:26:13,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742108_1284 (size=12595) 2024-11-26T15:26:13,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T15:26:13,175 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/A of 21fb246aea47afdec08e90541f5d61cc into a45d857666e44854ab97afd8f49b2a31(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:13,175 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:13,175 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/A, priority=10, startTime=1732634773135; duration=0sec 2024-11-26T15:26:13,175 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:13,175 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:A 2024-11-26T15:26:13,175 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-26T15:26:13,176 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/3a5d2502359a47d7a8537b4b7fd2083a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/3a5d2502359a47d7a8537b4b7fd2083a 2024-11-26T15:26:13,177 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72896 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-26T15:26:13,177 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/C is initiating minor compaction (all files) 2024-11-26T15:26:13,177 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/C in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:13,178 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/ebe9cef98bc14eee9118dcee66b38a42, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/688bdbcf1a274204aafd6f539de3c168, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/9691a598f17d4c76841e14b0684eebfd, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/39b7d4f653504bac9f680f72bb058a99, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/95743450405e4b0987797d70e395d6cf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/072d1c41c6be43db81678189dd004054] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=71.2 K 2024-11-26T15:26:13,178 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebe9cef98bc14eee9118dcee66b38a42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732634766876 2024-11-26T15:26:13,178 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 688bdbcf1a274204aafd6f539de3c168, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732634766999 2024-11-26T15:26:13,179 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9691a598f17d4c76841e14b0684eebfd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732634767654 2024-11-26T15:26:13,179 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39b7d4f653504bac9f680f72bb058a99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732634768314 2024-11-26T15:26:13,180 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95743450405e4b0987797d70e395d6cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732634769469 2024-11-26T15:26:13,180 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 072d1c41c6be43db81678189dd004054, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732634771676 2024-11-26T15:26:13,183 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/B of 21fb246aea47afdec08e90541f5d61cc into 3a5d2502359a47d7a8537b4b7fd2083a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:13,183 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:13,183 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/B, priority=10, startTime=1732634773135; duration=0sec 2024-11-26T15:26:13,183 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:13,183 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:B 2024-11-26T15:26:13,194 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#C#compaction#238 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:13,194 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/c85d2f43b83244d3a799fbf4499d0bd4 is 50, key is test_row_0/C:col10/1732634771676/Put/seqid=0 2024-11-26T15:26:13,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742110_1286 (size=12595) 2024-11-26T15:26:13,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T15:26:13,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:13,262 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-26T15:26:13,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/29ed7393364e435fb187792ac61f979f is 50, key is test_row_0/A:col10/1732634771812/Put/seqid=0 2024-11-26T15:26:13,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742111_1287 (size=12151) 2024-11-26T15:26:13,619 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/c85d2f43b83244d3a799fbf4499d0bd4 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/c85d2f43b83244d3a799fbf4499d0bd4 2024-11-26T15:26:13,632 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/C of 21fb246aea47afdec08e90541f5d61cc into c85d2f43b83244d3a799fbf4499d0bd4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:13,632 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:13,632 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/C, priority=10, startTime=1732634773135; duration=0sec 2024-11-26T15:26:13,633 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:13,633 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:C 2024-11-26T15:26:13,681 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/29ed7393364e435fb187792ac61f979f 2024-11-26T15:26:13,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/b56d17db292549938bff7098a48b7df7 is 50, key is test_row_0/B:col10/1732634771812/Put/seqid=0 2024-11-26T15:26:13,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742112_1288 (size=12151) 2024-11-26T15:26:13,751 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/b56d17db292549938bff7098a48b7df7 2024-11-26T15:26:13,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/174a3ee57c2c4339972c255493b7da49 is 50, key is test_row_0/C:col10/1732634771812/Put/seqid=0 2024-11-26T15:26:13,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742113_1289 (size=12151) 2024-11-26T15:26:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:13,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:14,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634834036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:14,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634834036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:14,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:14,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634834039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:14,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634834143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:14,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:14,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634834152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:14,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634834152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:14,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T15:26:14,220 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/174a3ee57c2c4339972c255493b7da49 2024-11-26T15:26:14,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/29ed7393364e435fb187792ac61f979f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/29ed7393364e435fb187792ac61f979f 2024-11-26T15:26:14,227 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/29ed7393364e435fb187792ac61f979f, entries=150, sequenceid=218, filesize=11.9 K 2024-11-26T15:26:14,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/b56d17db292549938bff7098a48b7df7 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/b56d17db292549938bff7098a48b7df7 2024-11-26T15:26:14,241 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/b56d17db292549938bff7098a48b7df7, entries=150, sequenceid=218, filesize=11.9 K 2024-11-26T15:26:14,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/174a3ee57c2c4339972c255493b7da49 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/174a3ee57c2c4339972c255493b7da49 2024-11-26T15:26:14,250 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/174a3ee57c2c4339972c255493b7da49, entries=150, sequenceid=218, filesize=11.9 K 2024-11-26T15:26:14,251 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 21fb246aea47afdec08e90541f5d61cc in 989ms, sequenceid=218, compaction requested=false 2024-11-26T15:26:14,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:14,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:14,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-26T15:26:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-26T15:26:14,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-26T15:26:14,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2170 sec 2024-11-26T15:26:14,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.2230 sec 2024-11-26T15:26:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:14,356 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:14,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-26T15:26:14,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/86a8543910694c26a96e747052dd0ce0 is 50, key is test_row_0/A:col10/1732634774034/Put/seqid=0 2024-11-26T15:26:14,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634834383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:14,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634834388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:14,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:14,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634834388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:14,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742114_1290 (size=12151) 2024-11-26T15:26:14,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/86a8543910694c26a96e747052dd0ce0 2024-11-26T15:26:14,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/1512d9a4d2e24ac4aa73507c4797465d is 50, key is test_row_0/B:col10/1732634774034/Put/seqid=0 2024-11-26T15:26:14,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742115_1291 (size=12151) 2024-11-26T15:26:14,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634834492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:14,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634834502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:14,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634834502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:14,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634834702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:14,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634834712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:14,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:14,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634834712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:14,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:14,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:14,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/1512d9a4d2e24ac4aa73507c4797465d 2024-11-26T15:26:14,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/6f733a4c9f32410498ebd03a49a05b6f is 50, key is test_row_0/C:col10/1732634774034/Put/seqid=0 2024-11-26T15:26:14,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742116_1292 (size=12151) 2024-11-26T15:26:14,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/6f733a4c9f32410498ebd03a49a05b6f 2024-11-26T15:26:14,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/86a8543910694c26a96e747052dd0ce0 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/86a8543910694c26a96e747052dd0ce0 2024-11-26T15:26:14,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/86a8543910694c26a96e747052dd0ce0, entries=150, sequenceid=246, filesize=11.9 K 2024-11-26T15:26:14,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/1512d9a4d2e24ac4aa73507c4797465d as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/1512d9a4d2e24ac4aa73507c4797465d 2024-11-26T15:26:14,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/1512d9a4d2e24ac4aa73507c4797465d, entries=150, sequenceid=246, filesize=11.9 K 2024-11-26T15:26:14,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/6f733a4c9f32410498ebd03a49a05b6f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/6f733a4c9f32410498ebd03a49a05b6f 2024-11-26T15:26:14,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/6f733a4c9f32410498ebd03a49a05b6f, entries=150, sequenceid=246, filesize=11.9 K 2024-11-26T15:26:14,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 21fb246aea47afdec08e90541f5d61cc in 538ms, sequenceid=246, compaction requested=true 2024-11-26T15:26:14,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:14,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:26:14,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:14,894 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:14,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:26:14,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:14,894 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:14,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:26:14,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:14,896 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:14,896 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/B is initiating minor compaction (all files) 2024-11-26T15:26:14,896 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/B in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:14,896 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/3a5d2502359a47d7a8537b4b7fd2083a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/b56d17db292549938bff7098a48b7df7, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/1512d9a4d2e24ac4aa73507c4797465d] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=36.0 K 2024-11-26T15:26:14,896 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:14,896 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/A is initiating minor compaction (all files) 2024-11-26T15:26:14,896 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/A in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:14,896 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a45d857666e44854ab97afd8f49b2a31, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/29ed7393364e435fb187792ac61f979f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/86a8543910694c26a96e747052dd0ce0] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=36.0 K 2024-11-26T15:26:14,897 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a5d2502359a47d7a8537b4b7fd2083a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732634771676 2024-11-26T15:26:14,897 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting a45d857666e44854ab97afd8f49b2a31, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732634771676 2024-11-26T15:26:14,897 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29ed7393364e435fb187792ac61f979f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732634771802 2024-11-26T15:26:14,897 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting b56d17db292549938bff7098a48b7df7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732634771802 2024-11-26T15:26:14,897 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86a8543910694c26a96e747052dd0ce0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732634774034 2024-11-26T15:26:14,897 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 1512d9a4d2e24ac4aa73507c4797465d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732634774034 2024-11-26T15:26:14,910 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#A#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:14,910 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/84d01f40f00549b4897d5fa26183549b is 50, key is test_row_0/A:col10/1732634774034/Put/seqid=0 2024-11-26T15:26:14,913 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#B#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:14,913 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/0a0667cab4ef46cc91db0c8671e9828e is 50, key is test_row_0/B:col10/1732634774034/Put/seqid=0 2024-11-26T15:26:14,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742117_1293 (size=12697) 2024-11-26T15:26:14,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742118_1294 (size=12697) 2024-11-26T15:26:14,931 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/84d01f40f00549b4897d5fa26183549b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/84d01f40f00549b4897d5fa26183549b 2024-11-26T15:26:14,932 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/0a0667cab4ef46cc91db0c8671e9828e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0a0667cab4ef46cc91db0c8671e9828e 2024-11-26T15:26:14,938 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/B of 21fb246aea47afdec08e90541f5d61cc into 0a0667cab4ef46cc91db0c8671e9828e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:14,938 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/A of 21fb246aea47afdec08e90541f5d61cc into 84d01f40f00549b4897d5fa26183549b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:14,938 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:14,938 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:14,938 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/B, priority=13, startTime=1732634774894; duration=0sec 2024-11-26T15:26:14,938 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/A, priority=13, startTime=1732634774894; duration=0sec 2024-11-26T15:26:14,938 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:14,938 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:A 2024-11-26T15:26:14,938 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:14,938 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:14,938 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:B 2024-11-26T15:26:14,940 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:14,940 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/C is initiating minor compaction (all files) 2024-11-26T15:26:14,940 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/C in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:14,940 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/c85d2f43b83244d3a799fbf4499d0bd4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/174a3ee57c2c4339972c255493b7da49, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/6f733a4c9f32410498ebd03a49a05b6f] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=36.0 K 2024-11-26T15:26:14,942 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting c85d2f43b83244d3a799fbf4499d0bd4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732634771676 2024-11-26T15:26:14,942 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 174a3ee57c2c4339972c255493b7da49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732634771802 2024-11-26T15:26:14,942 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f733a4c9f32410498ebd03a49a05b6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732634774034 2024-11-26T15:26:14,951 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#C#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:14,952 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/931ec6dd21d24b91b48c914c52a53ca4 is 50, key is test_row_0/C:col10/1732634774034/Put/seqid=0 2024-11-26T15:26:14,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742119_1295 (size=12697) 2024-11-26T15:26:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:15,017 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:15,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T15:26:15,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/66f108a8eae44873800e8293790742f3 is 50, key is test_row_0/A:col10/1732634774370/Put/seqid=0 2024-11-26T15:26:15,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742120_1296 (size=12201) 2024-11-26T15:26:15,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634835055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634835056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:15,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:15,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634835059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:15,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:15,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634835163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:15,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634835173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:15,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634835173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:15,364 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/931ec6dd21d24b91b48c914c52a53ca4 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/931ec6dd21d24b91b48c914c52a53ca4 2024-11-26T15:26:15,369 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/C of 21fb246aea47afdec08e90541f5d61cc into 931ec6dd21d24b91b48c914c52a53ca4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:15,369 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:15,369 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/C, priority=13, startTime=1732634774894; duration=0sec 2024-11-26T15:26:15,369 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:15,369 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:C 2024-11-26T15:26:15,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634835373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:15,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:15,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634835383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634835383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:15,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:15,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/66f108a8eae44873800e8293790742f3 2024-11-26T15:26:15,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/0d5aed8231a147aa82f81b0709af97f4 is 50, key is test_row_0/B:col10/1732634774370/Put/seqid=0 2024-11-26T15:26:15,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742121_1297 (size=12201) 2024-11-26T15:26:15,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/0d5aed8231a147aa82f81b0709af97f4 2024-11-26T15:26:15,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/8affdca4b5c04f048f65922ff5c3841b is 50, key is test_row_0/C:col10/1732634774370/Put/seqid=0 2024-11-26T15:26:15,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742122_1298 (size=12201) 2024-11-26T15:26:15,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/8affdca4b5c04f048f65922ff5c3841b 2024-11-26T15:26:15,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/66f108a8eae44873800e8293790742f3 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/66f108a8eae44873800e8293790742f3 2024-11-26T15:26:15,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/66f108a8eae44873800e8293790742f3, entries=150, sequenceid=259, filesize=11.9 K 2024-11-26T15:26:15,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/0d5aed8231a147aa82f81b0709af97f4 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0d5aed8231a147aa82f81b0709af97f4 2024-11-26T15:26:15,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0d5aed8231a147aa82f81b0709af97f4, entries=150, sequenceid=259, filesize=11.9 K 2024-11-26T15:26:15,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/8affdca4b5c04f048f65922ff5c3841b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/8affdca4b5c04f048f65922ff5c3841b 2024-11-26T15:26:15,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/8affdca4b5c04f048f65922ff5c3841b, entries=150, sequenceid=259, filesize=11.9 K 2024-11-26T15:26:15,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 21fb246aea47afdec08e90541f5d61cc in 521ms, sequenceid=259, compaction requested=false 2024-11-26T15:26:15,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:15,685 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:15,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-26T15:26:15,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:15,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:15,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634835700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634835700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634835701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/a1a8024dc39f43a380bcaa1b5c4b2861 is 50, key is test_row_0/A:col10/1732634775684/Put/seqid=0 2024-11-26T15:26:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:15,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:15,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742123_1299 (size=14741) 2024-11-26T15:26:15,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634835813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:15,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634835813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634835813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:16,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634836023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:16,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:16,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634836023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:16,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:16,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634836023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:16,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/a1a8024dc39f43a380bcaa1b5c4b2861 2024-11-26T15:26:16,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/0dd72609ed8944c6b1ac7fd3f2d990a5 is 50, key is test_row_0/B:col10/1732634775684/Put/seqid=0 2024-11-26T15:26:16,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742124_1300 (size=12301) 2024-11-26T15:26:16,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/0dd72609ed8944c6b1ac7fd3f2d990a5 2024-11-26T15:26:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T15:26:16,194 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:16,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-26T15:26:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T15:26:16,197 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:16,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/beaea8a402bf4dcb99b47f89c7057131 is 50, key is test_row_0/C:col10/1732634775684/Put/seqid=0 2024-11-26T15:26:16,200 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:16,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:16,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742125_1301 (size=12301) 2024-11-26T15:26:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T15:26:16,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634836333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:16,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634836333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:16,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634836333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:16,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T15:26:16,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:16,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:16,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:16,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:16,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:16,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T15:26:16,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:16,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:16,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:16,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:16,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T15:26:16,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/beaea8a402bf4dcb99b47f89c7057131 2024-11-26T15:26:16,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/a1a8024dc39f43a380bcaa1b5c4b2861 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a1a8024dc39f43a380bcaa1b5c4b2861 2024-11-26T15:26:16,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a1a8024dc39f43a380bcaa1b5c4b2861, entries=200, sequenceid=286, filesize=14.4 K 2024-11-26T15:26:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/0dd72609ed8944c6b1ac7fd3f2d990a5 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0dd72609ed8944c6b1ac7fd3f2d990a5 2024-11-26T15:26:16,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0dd72609ed8944c6b1ac7fd3f2d990a5, entries=150, sequenceid=286, filesize=12.0 K 2024-11-26T15:26:16,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/beaea8a402bf4dcb99b47f89c7057131 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/beaea8a402bf4dcb99b47f89c7057131 2024-11-26T15:26:16,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/beaea8a402bf4dcb99b47f89c7057131, entries=150, sequenceid=286, filesize=12.0 K 2024-11-26T15:26:16,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 21fb246aea47afdec08e90541f5d61cc in 964ms, sequenceid=286, compaction requested=true 2024-11-26T15:26:16,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:16,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:26:16,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:16,650 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:16,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:26:16,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:16,650 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:16,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:26:16,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:16,650 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39639 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:16,650 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:16,651 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/A is initiating minor compaction (all files) 2024-11-26T15:26:16,651 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/B is initiating minor compaction (all files) 2024-11-26T15:26:16,651 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/A in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:16,651 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/B in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:16,651 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/84d01f40f00549b4897d5fa26183549b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/66f108a8eae44873800e8293790742f3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a1a8024dc39f43a380bcaa1b5c4b2861] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=38.7 K 2024-11-26T15:26:16,651 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0a0667cab4ef46cc91db0c8671e9828e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0d5aed8231a147aa82f81b0709af97f4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0dd72609ed8944c6b1ac7fd3f2d990a5] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=36.3 K 2024-11-26T15:26:16,651 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84d01f40f00549b4897d5fa26183549b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732634774034 2024-11-26T15:26:16,651 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a0667cab4ef46cc91db0c8671e9828e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732634774034 2024-11-26T15:26:16,651 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d5aed8231a147aa82f81b0709af97f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732634774370 2024-11-26T15:26:16,651 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66f108a8eae44873800e8293790742f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732634774370 2024-11-26T15:26:16,652 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dd72609ed8944c6b1ac7fd3f2d990a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732634775048 2024-11-26T15:26:16,652 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1a8024dc39f43a380bcaa1b5c4b2861, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732634775048 2024-11-26T15:26:16,658 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#A#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:16,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T15:26:16,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:16,659 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/aea17605460e471dad01e1665909ce6c is 50, key is test_row_0/A:col10/1732634775684/Put/seqid=0 2024-11-26T15:26:16,659 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-26T15:26:16,666 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#B#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:16,666 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/c40bc4f2ec314f1693505f3967a791d2 is 50, key is test_row_0/B:col10/1732634775684/Put/seqid=0 2024-11-26T15:26:16,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/84e4329c20014cafb4cd64d4c61ab2b3 is 50, key is test_row_0/A:col10/1732634775698/Put/seqid=0 2024-11-26T15:26:16,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742126_1302 (size=12949) 2024-11-26T15:26:16,686 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/aea17605460e471dad01e1665909ce6c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/aea17605460e471dad01e1665909ce6c 2024-11-26T15:26:16,690 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/A of 21fb246aea47afdec08e90541f5d61cc into aea17605460e471dad01e1665909ce6c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:16,690 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:16,690 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/A, priority=13, startTime=1732634776650; duration=0sec 2024-11-26T15:26:16,690 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:16,690 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:A 2024-11-26T15:26:16,691 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:16,691 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:16,692 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/C is initiating minor compaction (all files) 2024-11-26T15:26:16,692 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/C in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:16,692 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/931ec6dd21d24b91b48c914c52a53ca4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/8affdca4b5c04f048f65922ff5c3841b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/beaea8a402bf4dcb99b47f89c7057131] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=36.3 K 2024-11-26T15:26:16,692 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 931ec6dd21d24b91b48c914c52a53ca4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732634774034 2024-11-26T15:26:16,692 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8affdca4b5c04f048f65922ff5c3841b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732634774370 2024-11-26T15:26:16,693 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting beaea8a402bf4dcb99b47f89c7057131, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732634775048 2024-11-26T15:26:16,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742128_1304 (size=12301) 2024-11-26T15:26:16,701 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#C#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:16,701 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/677afdbc96344dbb97d38ac16a4f7a10 is 50, key is test_row_0/C:col10/1732634775684/Put/seqid=0 2024-11-26T15:26:16,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742129_1305 (size=12949) 2024-11-26T15:26:16,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742127_1303 (size=12949) 2024-11-26T15:26:16,711 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/c40bc4f2ec314f1693505f3967a791d2 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/c40bc4f2ec314f1693505f3967a791d2 2024-11-26T15:26:16,716 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/B of 21fb246aea47afdec08e90541f5d61cc into c40bc4f2ec314f1693505f3967a791d2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:16,716 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:16,717 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/B, priority=13, startTime=1732634776650; duration=0sec 2024-11-26T15:26:16,717 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:16,717 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:B 2024-11-26T15:26:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T15:26:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:16,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:16,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634836884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:16,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:16,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634836885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:16,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:16,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634836885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:16,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:16,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634836992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:16,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634836992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:16,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634836992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:17,103 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/84e4329c20014cafb4cd64d4c61ab2b3 2024-11-26T15:26:17,117 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/677afdbc96344dbb97d38ac16a4f7a10 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/677afdbc96344dbb97d38ac16a4f7a10 2024-11-26T15:26:17,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/3d83fa6a67b146169e5e938bba1c0586 is 50, key is test_row_0/B:col10/1732634775698/Put/seqid=0 2024-11-26T15:26:17,124 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/C of 21fb246aea47afdec08e90541f5d61cc into 677afdbc96344dbb97d38ac16a4f7a10(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:17,125 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:17,125 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/C, priority=13, startTime=1732634776650; duration=0sec 2024-11-26T15:26:17,125 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:17,125 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:C 2024-11-26T15:26:17,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742130_1306 (size=12301) 2024-11-26T15:26:17,138 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/3d83fa6a67b146169e5e938bba1c0586 2024-11-26T15:26:17,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/8b6355ab1dc346b38d93b52ba75a7bf7 is 50, key is test_row_0/C:col10/1732634775698/Put/seqid=0 2024-11-26T15:26:17,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742131_1307 (size=12301) 2024-11-26T15:26:17,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:17,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634837203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634837203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:17,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634837203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T15:26:17,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:17,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634837512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:17,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:17,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:17,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:17,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634837513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:17,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:17,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:17,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:17,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:17,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:17,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634837513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:17,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:17,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:17,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:17,573 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/8b6355ab1dc346b38d93b52ba75a7bf7 2024-11-26T15:26:17,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/84e4329c20014cafb4cd64d4c61ab2b3 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/84e4329c20014cafb4cd64d4c61ab2b3 2024-11-26T15:26:17,587 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/84e4329c20014cafb4cd64d4c61ab2b3, entries=150, sequenceid=296, filesize=12.0 K 2024-11-26T15:26:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/3d83fa6a67b146169e5e938bba1c0586 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/3d83fa6a67b146169e5e938bba1c0586 2024-11-26T15:26:17,594 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/3d83fa6a67b146169e5e938bba1c0586, entries=150, sequenceid=296, filesize=12.0 K 2024-11-26T15:26:17,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/8b6355ab1dc346b38d93b52ba75a7bf7 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/8b6355ab1dc346b38d93b52ba75a7bf7 2024-11-26T15:26:17,601 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/8b6355ab1dc346b38d93b52ba75a7bf7, entries=150, sequenceid=296, filesize=12.0 K 2024-11-26T15:26:17,602 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 21fb246aea47afdec08e90541f5d61cc in 942ms, sequenceid=296, compaction requested=false 2024-11-26T15:26:17,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:17,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:17,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-26T15:26:17,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-26T15:26:17,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-26T15:26:17,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-11-26T15:26:17,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4150 sec 2024-11-26T15:26:18,026 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:18,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-26T15:26:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:18,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634838032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:18,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634838032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:18,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634838034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:18,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/17af15ba911a45a99156eda1ce26f189 is 50, key is test_row_0/A:col10/1732634778025/Put/seqid=0 2024-11-26T15:26:18,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742132_1308 (size=17181) 2024-11-26T15:26:18,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634838143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:18,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634838143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:18,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634838143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T15:26:18,343 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:18,344 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-26T15:26:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T15:26:18,347 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:18,348 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:18,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:18,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634838352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:18,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634838352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:18,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634838352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T15:26:18,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/17af15ba911a45a99156eda1ce26f189 2024-11-26T15:26:18,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/9c51359172394ac9b19fa6121d8ff5df is 50, key is test_row_0/B:col10/1732634778025/Put/seqid=0 2024-11-26T15:26:18,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T15:26:18,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:18,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:18,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:18,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:18,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:18,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:18,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742133_1309 (size=12301) 2024-11-26T15:26:18,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/9c51359172394ac9b19fa6121d8ff5df 2024-11-26T15:26:18,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/e7a75a9f61fa4adf8e05f11094fbc943 is 50, key is test_row_0/C:col10/1732634778025/Put/seqid=0 2024-11-26T15:26:18,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742134_1310 (size=12301) 2024-11-26T15:26:18,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/e7a75a9f61fa4adf8e05f11094fbc943 2024-11-26T15:26:18,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/17af15ba911a45a99156eda1ce26f189 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/17af15ba911a45a99156eda1ce26f189 2024-11-26T15:26:18,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/17af15ba911a45a99156eda1ce26f189, entries=250, sequenceid=329, filesize=16.8 K 2024-11-26T15:26:18,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/9c51359172394ac9b19fa6121d8ff5df as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/9c51359172394ac9b19fa6121d8ff5df 2024-11-26T15:26:18,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/9c51359172394ac9b19fa6121d8ff5df, entries=150, sequenceid=329, filesize=12.0 K 2024-11-26T15:26:18,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/e7a75a9f61fa4adf8e05f11094fbc943 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/e7a75a9f61fa4adf8e05f11094fbc943 2024-11-26T15:26:18,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/e7a75a9f61fa4adf8e05f11094fbc943, entries=150, sequenceid=329, filesize=12.0 K 2024-11-26T15:26:18,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 21fb246aea47afdec08e90541f5d61cc in 549ms, sequenceid=329, compaction requested=true 2024-11-26T15:26:18,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:18,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:26:18,575 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:18,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:18,575 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:18,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:26:18,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:18,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:26:18,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:18,578 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:18,578 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:18,578 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/B is initiating minor compaction (all files) 2024-11-26T15:26:18,578 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/A is initiating minor compaction (all files) 2024-11-26T15:26:18,578 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/B in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:18,578 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/A in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:18,578 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/c40bc4f2ec314f1693505f3967a791d2, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/3d83fa6a67b146169e5e938bba1c0586, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/9c51359172394ac9b19fa6121d8ff5df] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=36.7 K 2024-11-26T15:26:18,578 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/aea17605460e471dad01e1665909ce6c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/84e4329c20014cafb4cd64d4c61ab2b3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/17af15ba911a45a99156eda1ce26f189] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=41.4 K 2024-11-26T15:26:18,579 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting aea17605460e471dad01e1665909ce6c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732634775048 2024-11-26T15:26:18,579 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting c40bc4f2ec314f1693505f3967a791d2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732634775048 2024-11-26T15:26:18,579 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84e4329c20014cafb4cd64d4c61ab2b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732634775698 2024-11-26T15:26:18,579 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d83fa6a67b146169e5e938bba1c0586, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732634775698 2024-11-26T15:26:18,579 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17af15ba911a45a99156eda1ce26f189, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732634776873 2024-11-26T15:26:18,579 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c51359172394ac9b19fa6121d8ff5df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732634778024 2024-11-26T15:26:18,588 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#B#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:18,589 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/2f35bc4208a548b0a99ae319a720b455 is 50, key is test_row_0/B:col10/1732634778025/Put/seqid=0 2024-11-26T15:26:18,592 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#A#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:18,592 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/1a63438ec1734329b92b2ba07d6bcc67 is 50, key is test_row_0/A:col10/1732634778025/Put/seqid=0 2024-11-26T15:26:18,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742135_1311 (size=13051) 2024-11-26T15:26:18,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742136_1312 (size=13051) 2024-11-26T15:26:18,607 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/1a63438ec1734329b92b2ba07d6bcc67 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/1a63438ec1734329b92b2ba07d6bcc67 2024-11-26T15:26:18,607 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/2f35bc4208a548b0a99ae319a720b455 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/2f35bc4208a548b0a99ae319a720b455 2024-11-26T15:26:18,617 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/B of 21fb246aea47afdec08e90541f5d61cc into 2f35bc4208a548b0a99ae319a720b455(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:18,617 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/A of 21fb246aea47afdec08e90541f5d61cc into 1a63438ec1734329b92b2ba07d6bcc67(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:18,617 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:18,617 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/B, priority=13, startTime=1732634778575; duration=0sec 2024-11-26T15:26:18,617 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:18,617 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:B 2024-11-26T15:26:18,617 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:18,618 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:18,618 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/A, priority=13, startTime=1732634778575; duration=0sec 2024-11-26T15:26:18,618 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:18,618 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:A 2024-11-26T15:26:18,621 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:18,621 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/C is initiating minor compaction (all files) 2024-11-26T15:26:18,621 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/C in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:18,621 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/677afdbc96344dbb97d38ac16a4f7a10, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/8b6355ab1dc346b38d93b52ba75a7bf7, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/e7a75a9f61fa4adf8e05f11094fbc943] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=36.7 K 2024-11-26T15:26:18,622 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 677afdbc96344dbb97d38ac16a4f7a10, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732634775048 2024-11-26T15:26:18,623 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b6355ab1dc346b38d93b52ba75a7bf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732634775698 2024-11-26T15:26:18,625 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting e7a75a9f61fa4adf8e05f11094fbc943, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732634778024 2024-11-26T15:26:18,650 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#C#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:18,650 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/15cc8348541d46268d518b2e9136c5b2 is 50, key is test_row_0/C:col10/1732634778025/Put/seqid=0 2024-11-26T15:26:18,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T15:26:18,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:18,656 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-26T15:26:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T15:26:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:18,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:18,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/c20949bbe5c94b7e877a69b18e491dde is 50, key is test_row_0/A:col10/1732634778030/Put/seqid=0 2024-11-26T15:26:18,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742137_1313 (size=13051) 2024-11-26T15:26:18,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742138_1314 (size=9857) 2024-11-26T15:26:18,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634838724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:18,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:18,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634838729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:18,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634838730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:18,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634838832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:18,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634838842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:18,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634838842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T15:26:19,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634839042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:19,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634839052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634839052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:19,079 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/15cc8348541d46268d518b2e9136c5b2 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/15cc8348541d46268d518b2e9136c5b2 2024-11-26T15:26:19,080 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/c20949bbe5c94b7e877a69b18e491dde 2024-11-26T15:26:19,092 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/C of 21fb246aea47afdec08e90541f5d61cc into 15cc8348541d46268d518b2e9136c5b2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:19,092 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:19,092 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/C, priority=13, startTime=1732634778575; duration=0sec 2024-11-26T15:26:19,092 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:19,092 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:C 2024-11-26T15:26:19,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/35077b8f380f48ceaa4638c572dbea36 is 50, key is test_row_0/B:col10/1732634778030/Put/seqid=0 2024-11-26T15:26:19,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742139_1315 (size=9857) 2024-11-26T15:26:19,104 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/35077b8f380f48ceaa4638c572dbea36 2024-11-26T15:26:19,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/1253593379904d9198a9e41296124ac9 is 50, key is test_row_0/C:col10/1732634778030/Put/seqid=0 2024-11-26T15:26:19,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742140_1316 (size=9857) 2024-11-26T15:26:19,124 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/1253593379904d9198a9e41296124ac9 2024-11-26T15:26:19,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/c20949bbe5c94b7e877a69b18e491dde as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/c20949bbe5c94b7e877a69b18e491dde 2024-11-26T15:26:19,145 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/c20949bbe5c94b7e877a69b18e491dde, entries=100, sequenceid=337, filesize=9.6 K 2024-11-26T15:26:19,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/35077b8f380f48ceaa4638c572dbea36 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/35077b8f380f48ceaa4638c572dbea36 2024-11-26T15:26:19,155 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/35077b8f380f48ceaa4638c572dbea36, entries=100, sequenceid=337, filesize=9.6 K 2024-11-26T15:26:19,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/1253593379904d9198a9e41296124ac9 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/1253593379904d9198a9e41296124ac9 2024-11-26T15:26:19,164 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/1253593379904d9198a9e41296124ac9, entries=100, sequenceid=337, filesize=9.6 K 2024-11-26T15:26:19,165 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=187.85 KB/192360 for 21fb246aea47afdec08e90541f5d61cc in 509ms, sequenceid=337, compaction requested=false 2024-11-26T15:26:19,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:19,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:19,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-26T15:26:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-26T15:26:19,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-26T15:26:19,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 817 msec 2024-11-26T15:26:19,172 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 826 msec 2024-11-26T15:26:19,356 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:19,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=194.56 KB heapSize=510.52 KB 2024-11-26T15:26:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:19,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634839363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:19,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634839363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:19,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634839363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:19,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/1d71d7757bcd4b66a8038c66fa907842 is 50, key is test_row_0/A:col10/1732634779354/Put/seqid=0 2024-11-26T15:26:19,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742141_1317 (size=14741) 2024-11-26T15:26:19,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/1d71d7757bcd4b66a8038c66fa907842 2024-11-26T15:26:19,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/bf9a252cedad4c34b79a5d2b4229c82c is 50, key is test_row_0/B:col10/1732634779354/Put/seqid=0 2024-11-26T15:26:19,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742142_1318 (size=12301) 2024-11-26T15:26:19,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/bf9a252cedad4c34b79a5d2b4229c82c 2024-11-26T15:26:19,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/c3cb65ea7279459294fbc87ac8a30848 is 50, key is test_row_0/C:col10/1732634779354/Put/seqid=0 2024-11-26T15:26:19,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742143_1319 (size=12301) 2024-11-26T15:26:19,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634839472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T15:26:19,488 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:19,489 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-26T15:26:19,492 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T15:26:19,493 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:19,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T15:26:19,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T15:26:19,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:19,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:19,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:19,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:19,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:19,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634839682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:19,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T15:26:19,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:19,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:19,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:19,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:19,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T15:26:19,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/c3cb65ea7279459294fbc87ac8a30848 2024-11-26T15:26:19,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/1d71d7757bcd4b66a8038c66fa907842 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/1d71d7757bcd4b66a8038c66fa907842 2024-11-26T15:26:19,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/1d71d7757bcd4b66a8038c66fa907842, entries=200, sequenceid=370, filesize=14.4 K 2024-11-26T15:26:19,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/bf9a252cedad4c34b79a5d2b4229c82c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/bf9a252cedad4c34b79a5d2b4229c82c 2024-11-26T15:26:19,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634839872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634839872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:19,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/bf9a252cedad4c34b79a5d2b4229c82c, entries=150, sequenceid=370, filesize=12.0 K 2024-11-26T15:26:19,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/c3cb65ea7279459294fbc87ac8a30848 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/c3cb65ea7279459294fbc87ac8a30848 2024-11-26T15:26:19,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/c3cb65ea7279459294fbc87ac8a30848, entries=150, sequenceid=370, filesize=12.0 K 2024-11-26T15:26:19,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=6.71 KB/6870 for 21fb246aea47afdec08e90541f5d61cc in 528ms, sequenceid=370, compaction requested=true 2024-11-26T15:26:19,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:19,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:26:19,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:19,885 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:19,885 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:19,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:26:19,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:19,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:26:19,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:19,886 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:19,886 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/A is initiating minor compaction (all files) 2024-11-26T15:26:19,886 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/A in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:19,886 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/1a63438ec1734329b92b2ba07d6bcc67, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/c20949bbe5c94b7e877a69b18e491dde, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/1d71d7757bcd4b66a8038c66fa907842] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=36.8 K 2024-11-26T15:26:19,886 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:19,887 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/B is initiating minor compaction (all files) 2024-11-26T15:26:19,887 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/B in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:19,887 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/2f35bc4208a548b0a99ae319a720b455, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/35077b8f380f48ceaa4638c572dbea36, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/bf9a252cedad4c34b79a5d2b4229c82c] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=34.4 K 2024-11-26T15:26:19,888 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f35bc4208a548b0a99ae319a720b455, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732634778024 2024-11-26T15:26:19,888 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a63438ec1734329b92b2ba07d6bcc67, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732634778024 2024-11-26T15:26:19,889 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 35077b8f380f48ceaa4638c572dbea36, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732634778030 2024-11-26T15:26:19,889 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting c20949bbe5c94b7e877a69b18e491dde, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732634778030 2024-11-26T15:26:19,889 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting bf9a252cedad4c34b79a5d2b4229c82c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732634778721 2024-11-26T15:26:19,889 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d71d7757bcd4b66a8038c66fa907842, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732634778721 2024-11-26T15:26:19,914 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#B#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:19,914 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/844edb36fdaa4dec819ad656bda7717b is 50, key is test_row_0/B:col10/1732634779354/Put/seqid=0 2024-11-26T15:26:19,914 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#A#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:19,915 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/15f07c1c60f240d4ad084f8402568059 is 50, key is test_row_0/A:col10/1732634779354/Put/seqid=0 2024-11-26T15:26:19,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742144_1320 (size=13153) 2024-11-26T15:26:19,952 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/15f07c1c60f240d4ad084f8402568059 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/15f07c1c60f240d4ad084f8402568059 2024-11-26T15:26:19,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T15:26:19,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:19,953 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-26T15:26:19,960 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/A of 21fb246aea47afdec08e90541f5d61cc into 15f07c1c60f240d4ad084f8402568059(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:19,960 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:19,960 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/A, priority=13, startTime=1732634779885; duration=0sec 2024-11-26T15:26:19,960 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:19,960 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:A 2024-11-26T15:26:19,960 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:19,961 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:19,962 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/C is initiating minor compaction (all files) 2024-11-26T15:26:19,962 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/C in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:19,962 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/15cc8348541d46268d518b2e9136c5b2, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/1253593379904d9198a9e41296124ac9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/c3cb65ea7279459294fbc87ac8a30848] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=34.4 K 2024-11-26T15:26:19,962 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15cc8348541d46268d518b2e9136c5b2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732634778024 2024-11-26T15:26:19,963 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1253593379904d9198a9e41296124ac9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732634778030 2024-11-26T15:26:19,963 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3cb65ea7279459294fbc87ac8a30848, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732634778721 2024-11-26T15:26:19,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/addc074fd89f49579a6c3db982f3900f is 50, key is test_row_1/A:col10/1732634779361/Put/seqid=0 2024-11-26T15:26:19,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742145_1321 (size=13153) 2024-11-26T15:26:19,973 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#C#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:19,974 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/61f4d8ce9f974459ba8446e3588fe5d9 is 50, key is test_row_0/C:col10/1732634779354/Put/seqid=0 2024-11-26T15:26:19,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742146_1322 (size=7415) 2024-11-26T15:26:19,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742147_1323 (size=13153) 2024-11-26T15:26:20,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:20,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634840119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T15:26:20,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634840233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:20,382 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/844edb36fdaa4dec819ad656bda7717b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/844edb36fdaa4dec819ad656bda7717b 2024-11-26T15:26:20,394 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/B of 21fb246aea47afdec08e90541f5d61cc into 844edb36fdaa4dec819ad656bda7717b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:20,394 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/addc074fd89f49579a6c3db982f3900f 2024-11-26T15:26:20,394 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:20,394 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/B, priority=13, startTime=1732634779885; duration=0sec 2024-11-26T15:26:20,395 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:20,395 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:B 2024-11-26T15:26:20,401 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/61f4d8ce9f974459ba8446e3588fe5d9 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/61f4d8ce9f974459ba8446e3588fe5d9 2024-11-26T15:26:20,406 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/C of 21fb246aea47afdec08e90541f5d61cc into 61f4d8ce9f974459ba8446e3588fe5d9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:20,406 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:20,406 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/C, priority=13, startTime=1732634779885; duration=0sec 2024-11-26T15:26:20,406 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:20,406 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:C 2024-11-26T15:26:20,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/8aa068ee2f8641e0a061fa4cecd8a30d is 50, key is test_row_1/B:col10/1732634779361/Put/seqid=0 2024-11-26T15:26:20,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742148_1324 (size=7415) 2024-11-26T15:26:20,442 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/8aa068ee2f8641e0a061fa4cecd8a30d 2024-11-26T15:26:20,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634840443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:20,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/6d6a43ee35484434a77d1d0360ea857e is 50, key is test_row_1/C:col10/1732634779361/Put/seqid=0 2024-11-26T15:26:20,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742149_1325 (size=7415) 2024-11-26T15:26:20,488 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/6d6a43ee35484434a77d1d0360ea857e 2024-11-26T15:26:20,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/addc074fd89f49579a6c3db982f3900f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/addc074fd89f49579a6c3db982f3900f 2024-11-26T15:26:20,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33556 deadline: 1732634840493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:20,495 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in 21fb246aea47afdec08e90541f5d61cc of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18282 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:20,498 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/addc074fd89f49579a6c3db982f3900f, entries=50, sequenceid=374, filesize=7.2 K 2024-11-26T15:26:20,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/8aa068ee2f8641e0a061fa4cecd8a30d as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/8aa068ee2f8641e0a061fa4cecd8a30d 2024-11-26T15:26:20,506 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/8aa068ee2f8641e0a061fa4cecd8a30d, entries=50, sequenceid=374, filesize=7.2 K 2024-11-26T15:26:20,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/6d6a43ee35484434a77d1d0360ea857e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/6d6a43ee35484434a77d1d0360ea857e 2024-11-26T15:26:20,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33610 deadline: 1732634840512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:20,514 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/6d6a43ee35484434a77d1d0360ea857e, entries=50, sequenceid=374, filesize=7.2 K 2024-11-26T15:26:20,514 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in 21fb246aea47afdec08e90541f5d61cc of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18301 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:20,515 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=194.56 KB/199230 for 21fb246aea47afdec08e90541f5d61cc in 562ms, sequenceid=374, compaction requested=false 2024-11-26T15:26:20,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:20,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:20,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-26T15:26:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-26T15:26:20,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-26T15:26:20,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0250 sec 2024-11-26T15:26:20,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0340 sec 2024-11-26T15:26:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T15:26:20,633 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:20,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-26T15:26:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T15:26:20,636 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:20,637 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:20,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T15:26:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:20,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:20,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=201.27 KB heapSize=528.09 KB 2024-11-26T15:26:20,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634840756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:20,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/512e8f8c64b94e4c9ea8838b9746a5b5 is 50, key is test_row_0/A:col10/1732634780753/Put/seqid=0 2024-11-26T15:26:20,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742150_1326 (size=12301) 2024-11-26T15:26:20,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T15:26:20,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:20,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:20,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:20,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:20,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:20,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:20,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634840862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:20,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:20,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:20,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634840882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:20,884 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in 21fb246aea47afdec08e90541f5d61cc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:20,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634840893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:20,894 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in 21fb246aea47afdec08e90541f5d61cc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:20,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:20,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T15:26:20,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:20,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:20,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:20,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:20,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T15:26:21,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634841072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:21,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:21,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:21,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:21,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T15:26:21,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:21,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:21,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:21,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:21,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:21,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/512e8f8c64b94e4c9ea8838b9746a5b5 2024-11-26T15:26:21,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/22b413650d5c4a4996f0337c18bb29ba is 50, key is test_row_0/B:col10/1732634780753/Put/seqid=0 2024-11-26T15:26:21,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742151_1327 (size=12301) 2024-11-26T15:26:21,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/22b413650d5c4a4996f0337c18bb29ba 2024-11-26T15:26:21,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/27d6573e3e844b7fab822d33b3e24788 is 50, key is test_row_0/C:col10/1732634780753/Put/seqid=0 2024-11-26T15:26:21,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742152_1328 (size=12301) 2024-11-26T15:26:21,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T15:26:21,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:21,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:21,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:21,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:21,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T15:26:21,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634841383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:21,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:21,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:21,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:21,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T15:26:21,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:21,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:21,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:21,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:21,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:21,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T15:26:21,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:21,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:21,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:21,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:21,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:21,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/27d6573e3e844b7fab822d33b3e24788 2024-11-26T15:26:21,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/512e8f8c64b94e4c9ea8838b9746a5b5 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/512e8f8c64b94e4c9ea8838b9746a5b5 2024-11-26T15:26:21,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/512e8f8c64b94e4c9ea8838b9746a5b5, entries=150, sequenceid=410, filesize=12.0 K 2024-11-26T15:26:21,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/22b413650d5c4a4996f0337c18bb29ba as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/22b413650d5c4a4996f0337c18bb29ba 2024-11-26T15:26:21,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/22b413650d5c4a4996f0337c18bb29ba, entries=150, sequenceid=410, filesize=12.0 K 2024-11-26T15:26:21,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/27d6573e3e844b7fab822d33b3e24788 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/27d6573e3e844b7fab822d33b3e24788 2024-11-26T15:26:21,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/27d6573e3e844b7fab822d33b3e24788, entries=150, sequenceid=410, filesize=12.0 K 2024-11-26T15:26:21,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for 21fb246aea47afdec08e90541f5d61cc in 898ms, sequenceid=410, compaction requested=true 2024-11-26T15:26:21,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:21,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:26:21,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:21,653 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:21,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:26:21,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:21,653 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:21,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21fb246aea47afdec08e90541f5d61cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:26:21,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:21,654 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:21,654 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:21,654 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/B is initiating minor compaction (all files) 2024-11-26T15:26:21,654 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/A is initiating minor compaction (all files) 2024-11-26T15:26:21,655 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/A in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:21,655 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/B in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:21,655 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/15f07c1c60f240d4ad084f8402568059, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/addc074fd89f49579a6c3db982f3900f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/512e8f8c64b94e4c9ea8838b9746a5b5] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=32.1 K 2024-11-26T15:26:21,655 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/844edb36fdaa4dec819ad656bda7717b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/8aa068ee2f8641e0a061fa4cecd8a30d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/22b413650d5c4a4996f0337c18bb29ba] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=32.1 K 2024-11-26T15:26:21,655 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15f07c1c60f240d4ad084f8402568059, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732634778721 2024-11-26T15:26:21,655 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 844edb36fdaa4dec819ad656bda7717b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732634778721 2024-11-26T15:26:21,655 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aa068ee2f8641e0a061fa4cecd8a30d, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732634779361 2024-11-26T15:26:21,655 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting addc074fd89f49579a6c3db982f3900f, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732634779361 2024-11-26T15:26:21,655 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 22b413650d5c4a4996f0337c18bb29ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732634780089 2024-11-26T15:26:21,655 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 512e8f8c64b94e4c9ea8838b9746a5b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732634780089 2024-11-26T15:26:21,661 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#A#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:21,662 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/9e71c16b0edf4cfab800dce4f7d70f1c is 50, key is test_row_0/A:col10/1732634780753/Put/seqid=0 2024-11-26T15:26:21,664 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#B#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:21,664 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/7ceece3900f343e88bff767b27f24cee is 50, key is test_row_0/B:col10/1732634780753/Put/seqid=0 2024-11-26T15:26:21,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742154_1330 (size=13255) 2024-11-26T15:26:21,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742153_1329 (size=13255) 2024-11-26T15:26:21,698 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/9e71c16b0edf4cfab800dce4f7d70f1c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/9e71c16b0edf4cfab800dce4f7d70f1c 2024-11-26T15:26:21,704 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/A of 21fb246aea47afdec08e90541f5d61cc into 9e71c16b0edf4cfab800dce4f7d70f1c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:21,704 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:21,704 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/A, priority=13, startTime=1732634781653; duration=0sec 2024-11-26T15:26:21,704 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:21,704 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:A 2024-11-26T15:26:21,704 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:21,705 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:21,705 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): 21fb246aea47afdec08e90541f5d61cc/C is initiating minor compaction (all files) 2024-11-26T15:26:21,705 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21fb246aea47afdec08e90541f5d61cc/C in TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:21,705 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/61f4d8ce9f974459ba8446e3588fe5d9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/6d6a43ee35484434a77d1d0360ea857e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/27d6573e3e844b7fab822d33b3e24788] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp, totalSize=32.1 K 2024-11-26T15:26:21,706 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61f4d8ce9f974459ba8446e3588fe5d9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732634778721 2024-11-26T15:26:21,706 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d6a43ee35484434a77d1d0360ea857e, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732634779361 2024-11-26T15:26:21,706 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27d6573e3e844b7fab822d33b3e24788, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732634780089 2024-11-26T15:26:21,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T15:26:21,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:21,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:21,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:21,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-26T15:26:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-26T15:26:21,714 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21fb246aea47afdec08e90541f5d61cc#C#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:21,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-26T15:26:21,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0750 sec 2024-11-26T15:26:21,714 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/e00a0b1d2dc8466280ed565bf391ac0c is 50, key is test_row_0/C:col10/1732634780753/Put/seqid=0 2024-11-26T15:26:21,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0810 sec 2024-11-26T15:26:21,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742155_1331 (size=13255) 2024-11-26T15:26:21,733 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/e00a0b1d2dc8466280ed565bf391ac0c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/e00a0b1d2dc8466280ed565bf391ac0c 2024-11-26T15:26:21,738 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/C of 21fb246aea47afdec08e90541f5d61cc into e00a0b1d2dc8466280ed565bf391ac0c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:21,738 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:21,738 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/C, priority=13, startTime=1732634781653; duration=0sec 2024-11-26T15:26:21,739 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:21,739 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:C 2024-11-26T15:26:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T15:26:21,773 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:21,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-26T15:26:21,775 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T15:26:21,775 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:21,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T15:26:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:21,906 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:21,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T15:26:21,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/f1cf0b48a1664765998f6847b35c3230 is 50, key is test_row_0/A:col10/1732634781898/Put/seqid=0 2024-11-26T15:26:21,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742156_1332 (size=14741) 2024-11-26T15:26:21,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-26T15:26:21,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:21,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:21,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:21,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:21,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:21,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634841980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:21,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:21,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:21,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:22,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-26T15:26:22,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:22,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:22,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:22,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,085 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/7ceece3900f343e88bff767b27f24cee as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/7ceece3900f343e88bff767b27f24cee 2024-11-26T15:26:22,089 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21fb246aea47afdec08e90541f5d61cc/B of 21fb246aea47afdec08e90541f5d61cc into 7ceece3900f343e88bff767b27f24cee(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:22,089 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:22,089 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., storeName=21fb246aea47afdec08e90541f5d61cc/B, priority=13, startTime=1732634781653; duration=0sec 2024-11-26T15:26:22,089 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:22,089 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21fb246aea47afdec08e90541f5d61cc:B 2024-11-26T15:26:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T15:26:22,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634842092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:22,171 INFO [Thread-1149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1149. 2024-11-26T15:26:22,171 INFO [Thread-1143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1143. 2024-11-26T15:26:22,171 DEBUG [Thread-1143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:22,171 DEBUG [Thread-1149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:22,171 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:22,171 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:22,171 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:22,171 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:22,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:22,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:22,172 INFO [Thread-1147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1147. 2024-11-26T15:26:22,172 DEBUG [Thread-1147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:22,172 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:22,172 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:22,172 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:22,173 INFO [Thread-1145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1145. 2024-11-26T15:26:22,173 DEBUG [Thread-1145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:22,173 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:22,174 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:22,174 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:22,174 INFO [Thread-1151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1151. 2024-11-26T15:26:22,174 DEBUG [Thread-1151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:22,174 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:22,174 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:22,174 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:22,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-26T15:26:22,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:22,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:22,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:22,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634842303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:22,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/f1cf0b48a1664765998f6847b35c3230 2024-11-26T15:26:22,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/d43e98b34d7d4997aae4032f724118de is 50, key is test_row_0/B:col10/1732634781898/Put/seqid=0 2024-11-26T15:26:22,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742157_1333 (size=12301) 2024-11-26T15:26:22,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-26T15:26:22,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:22,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:22,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:22,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T15:26:22,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-26T15:26:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:22,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634842613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:22,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-26T15:26:22,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:22,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:22,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:22,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/d43e98b34d7d4997aae4032f724118de 2024-11-26T15:26:22,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/c950b24e685143349076ee38bfcddb9f is 50, key is test_row_0/C:col10/1732634781898/Put/seqid=0 2024-11-26T15:26:22,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742158_1334 (size=12301) 2024-11-26T15:26:22,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-26T15:26:22,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:22,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:22,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:22,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:22,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1732634842892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:22,893 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in 21fb246aea47afdec08e90541f5d61cc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:22,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33554 deadline: 1732634842902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:22,903 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in 21fb246aea47afdec08e90541f5d61cc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T15:26:23,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-26T15:26:23,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:23,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:23,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:23,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:23,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:23,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33598 deadline: 1732634843123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:26:23,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/c950b24e685143349076ee38bfcddb9f 2024-11-26T15:26:23,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/f1cf0b48a1664765998f6847b35c3230 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/f1cf0b48a1664765998f6847b35c3230 2024-11-26T15:26:23,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/f1cf0b48a1664765998f6847b35c3230, entries=200, sequenceid=424, filesize=14.4 K 2024-11-26T15:26:23,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/d43e98b34d7d4997aae4032f724118de as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/d43e98b34d7d4997aae4032f724118de 2024-11-26T15:26:23,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-26T15:26:23,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:23,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:23,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:23,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:23,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:23,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/d43e98b34d7d4997aae4032f724118de, entries=150, sequenceid=424, filesize=12.0 K 2024-11-26T15:26:23,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/c950b24e685143349076ee38bfcddb9f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/c950b24e685143349076ee38bfcddb9f 2024-11-26T15:26:23,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/c950b24e685143349076ee38bfcddb9f, entries=150, sequenceid=424, filesize=12.0 K 2024-11-26T15:26:23,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 21fb246aea47afdec08e90541f5d61cc in 1268ms, sequenceid=424, compaction requested=false 2024-11-26T15:26:23,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:23,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-26T15:26:23,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:23,322 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-26T15:26:23,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/bf2590d9b69c45ef82ecd976be2c8066 is 50, key is test_row_0/A:col10/1732634781968/Put/seqid=0 2024-11-26T15:26:23,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742159_1335 (size=12301) 2024-11-26T15:26:23,738 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/bf2590d9b69c45ef82ecd976be2c8066 2024-11-26T15:26:23,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/571ebf6541b444c5967d581e73425a54 is 50, key is test_row_0/B:col10/1732634781968/Put/seqid=0 2024-11-26T15:26:23,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742160_1336 (size=12301) 2024-11-26T15:26:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T15:26:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:24,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. as already flushing 2024-11-26T15:26:24,136 INFO [Thread-1134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1134. 2024-11-26T15:26:24,136 DEBUG [Thread-1134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:24,136 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:24,137 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:24,137 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:24,158 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/571ebf6541b444c5967d581e73425a54 2024-11-26T15:26:24,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/cb7680ce7f134672b12da3969eb112a0 is 50, key is test_row_0/C:col10/1732634781968/Put/seqid=0 2024-11-26T15:26:24,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742161_1337 (size=12301) 2024-11-26T15:26:24,578 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/cb7680ce7f134672b12da3969eb112a0 2024-11-26T15:26:24,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/bf2590d9b69c45ef82ecd976be2c8066 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/bf2590d9b69c45ef82ecd976be2c8066 2024-11-26T15:26:24,592 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/bf2590d9b69c45ef82ecd976be2c8066, entries=150, sequenceid=450, filesize=12.0 K 2024-11-26T15:26:24,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/571ebf6541b444c5967d581e73425a54 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/571ebf6541b444c5967d581e73425a54 2024-11-26T15:26:24,598 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/571ebf6541b444c5967d581e73425a54, entries=150, sequenceid=450, filesize=12.0 K 2024-11-26T15:26:24,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/cb7680ce7f134672b12da3969eb112a0 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/cb7680ce7f134672b12da3969eb112a0 2024-11-26T15:26:24,602 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/cb7680ce7f134672b12da3969eb112a0, entries=150, sequenceid=450, filesize=12.0 K 2024-11-26T15:26:24,603 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=6.71 KB/6870 for 21fb246aea47afdec08e90541f5d61cc in 1281ms, sequenceid=450, compaction requested=true 2024-11-26T15:26:24,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 21fb246aea47afdec08e90541f5d61cc: 2024-11-26T15:26:24,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:24,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-26T15:26:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-26T15:26:24,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-26T15:26:24,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8290 sec 2024-11-26T15:26:24,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.8330 sec 2024-11-26T15:26:25,237 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T15:26:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T15:26:25,933 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:26,916 INFO [Thread-1136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1136. 2024-11-26T15:26:26,916 DEBUG [Thread-1136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:26,916 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:26,917 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:26,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:26,934 INFO [Thread-1132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1132. 2024-11-26T15:26:26,934 DEBUG [Thread-1132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:26,934 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:26,934 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:26,934 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:30,536 INFO [Thread-1138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1138. 2024-11-26T15:26:30,536 DEBUG [Thread-1138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:30,536 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:30,537 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:30,537 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:30,584 INFO [Thread-1140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1140. 2024-11-26T15:26:30,584 DEBUG [Thread-1140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:30,584 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:30,584 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:30,585 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:30,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T15:26:30,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-26T15:26:30,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 162 2024-11-26T15:26:30,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-26T15:26:30,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 6 2024-11-26T15:26:30,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-26T15:26:30,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T15:26:30,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3904 2024-11-26T15:26:30,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3750 2024-11-26T15:26:30,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3985 2024-11-26T15:26:30,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3933 2024-11-26T15:26:30,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3752 2024-11-26T15:26:30,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T15:26:30,585 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T15:26:30,585 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T15:26:30,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:30,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:30,585 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:30,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T15:26:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T15:26:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-26T15:26:30,590 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634790590"}]},"ts":"1732634790590"} 2024-11-26T15:26:30,592 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T15:26:30,592 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T15:26:30,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T15:26:30,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21fb246aea47afdec08e90541f5d61cc, UNASSIGN}] 2024-11-26T15:26:30,595 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21fb246aea47afdec08e90541f5d61cc, UNASSIGN 2024-11-26T15:26:30,596 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=21fb246aea47afdec08e90541f5d61cc, regionState=CLOSING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:30,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21fb246aea47afdec08e90541f5d61cc, UNASSIGN because future has completed 2024-11-26T15:26:30,598 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T15:26:30,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:26:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-26T15:26:30,752 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:30,752 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T15:26:30,752 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 21fb246aea47afdec08e90541f5d61cc, disabling compactions & flushes 2024-11-26T15:26:30,752 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:30,752 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:30,752 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. after waiting 0 ms 2024-11-26T15:26:30,752 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:30,753 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 21fb246aea47afdec08e90541f5d61cc 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-26T15:26:30,760 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/58db72b8a0bc423696ba82bb6d05d97e is 50, key is test_row_0/A:col10/1732634790583/Put/seqid=0 2024-11-26T15:26:30,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742162_1338 (size=9857) 2024-11-26T15:26:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-26T15:26:31,166 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/58db72b8a0bc423696ba82bb6d05d97e 2024-11-26T15:26:31,179 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/bc6a67602a0345fc929e0d8ee4b1134e is 50, key is test_row_0/B:col10/1732634790583/Put/seqid=0 2024-11-26T15:26:31,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742163_1339 (size=9857) 2024-11-26T15:26:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-26T15:26:31,585 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/bc6a67602a0345fc929e0d8ee4b1134e 2024-11-26T15:26:31,597 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/77efa1bfac6b4552bdf10591cd59d512 is 50, key is test_row_0/C:col10/1732634790583/Put/seqid=0 2024-11-26T15:26:31,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742164_1340 (size=9857) 2024-11-26T15:26:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-26T15:26:32,002 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/77efa1bfac6b4552bdf10591cd59d512 2024-11-26T15:26:32,007 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/A/58db72b8a0bc423696ba82bb6d05d97e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/58db72b8a0bc423696ba82bb6d05d97e 2024-11-26T15:26:32,012 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/58db72b8a0bc423696ba82bb6d05d97e, entries=100, sequenceid=458, filesize=9.6 K 2024-11-26T15:26:32,013 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/B/bc6a67602a0345fc929e0d8ee4b1134e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/bc6a67602a0345fc929e0d8ee4b1134e 2024-11-26T15:26:32,017 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/bc6a67602a0345fc929e0d8ee4b1134e, entries=100, sequenceid=458, filesize=9.6 K 2024-11-26T15:26:32,018 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/.tmp/C/77efa1bfac6b4552bdf10591cd59d512 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/77efa1bfac6b4552bdf10591cd59d512 2024-11-26T15:26:32,021 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/77efa1bfac6b4552bdf10591cd59d512, entries=100, sequenceid=458, filesize=9.6 K 2024-11-26T15:26:32,021 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 21fb246aea47afdec08e90541f5d61cc in 1269ms, sequenceid=458, compaction requested=true 2024-11-26T15:26:32,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/fb5ecc43468e4529b3e07c8661672cd0, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/7734086310bb458492801a833cd71a85, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/e823b65524bc4d4c9a2340a0a022c255, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/eb1445d68e604d08a6d050a32544d509, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/bb970c5d57a84a6cb672d71729041891, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a9d89a42cdeb46ccb87ec8340baef9c1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/f0630962497a49f48f10caad7febc5a7, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/4820f58262c445fa8c1190f92c0b650e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/ed83b7ef7e01454a91dd3aa88292572f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/5aca6befd0a84d3d8f2015a01b2d1639, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/cf29669c2f314b9c9c254535fd75e88f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/6b2c645068ef434fb52c8b8940da79ba, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/2e1faaff441040fd860a62a9ab8e1331, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a45d857666e44854ab97afd8f49b2a31, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/29ed7393364e435fb187792ac61f979f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/84d01f40f00549b4897d5fa26183549b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/86a8543910694c26a96e747052dd0ce0, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/66f108a8eae44873800e8293790742f3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a1a8024dc39f43a380bcaa1b5c4b2861, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/aea17605460e471dad01e1665909ce6c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/84e4329c20014cafb4cd64d4c61ab2b3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/17af15ba911a45a99156eda1ce26f189, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/1a63438ec1734329b92b2ba07d6bcc67, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/c20949bbe5c94b7e877a69b18e491dde, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/1d71d7757bcd4b66a8038c66fa907842, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/15f07c1c60f240d4ad084f8402568059, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/addc074fd89f49579a6c3db982f3900f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/512e8f8c64b94e4c9ea8838b9746a5b5] to archive 2024-11-26T15:26:32,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:26:32,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/fb5ecc43468e4529b3e07c8661672cd0 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/fb5ecc43468e4529b3e07c8661672cd0 2024-11-26T15:26:32,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/7734086310bb458492801a833cd71a85 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/7734086310bb458492801a833cd71a85 2024-11-26T15:26:32,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/e823b65524bc4d4c9a2340a0a022c255 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/e823b65524bc4d4c9a2340a0a022c255 2024-11-26T15:26:32,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/eb1445d68e604d08a6d050a32544d509 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/eb1445d68e604d08a6d050a32544d509 2024-11-26T15:26:32,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/bb970c5d57a84a6cb672d71729041891 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/bb970c5d57a84a6cb672d71729041891 2024-11-26T15:26:32,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a9d89a42cdeb46ccb87ec8340baef9c1 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a9d89a42cdeb46ccb87ec8340baef9c1 2024-11-26T15:26:32,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/f0630962497a49f48f10caad7febc5a7 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/f0630962497a49f48f10caad7febc5a7 2024-11-26T15:26:32,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/4820f58262c445fa8c1190f92c0b650e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/4820f58262c445fa8c1190f92c0b650e 2024-11-26T15:26:32,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/ed83b7ef7e01454a91dd3aa88292572f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/ed83b7ef7e01454a91dd3aa88292572f 2024-11-26T15:26:32,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/5aca6befd0a84d3d8f2015a01b2d1639 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/5aca6befd0a84d3d8f2015a01b2d1639 2024-11-26T15:26:32,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/cf29669c2f314b9c9c254535fd75e88f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/cf29669c2f314b9c9c254535fd75e88f 2024-11-26T15:26:32,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/6b2c645068ef434fb52c8b8940da79ba to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/6b2c645068ef434fb52c8b8940da79ba 2024-11-26T15:26:32,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/2e1faaff441040fd860a62a9ab8e1331 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/2e1faaff441040fd860a62a9ab8e1331 2024-11-26T15:26:32,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a45d857666e44854ab97afd8f49b2a31 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a45d857666e44854ab97afd8f49b2a31 2024-11-26T15:26:32,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/29ed7393364e435fb187792ac61f979f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/29ed7393364e435fb187792ac61f979f 2024-11-26T15:26:32,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/84d01f40f00549b4897d5fa26183549b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/84d01f40f00549b4897d5fa26183549b 2024-11-26T15:26:32,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/86a8543910694c26a96e747052dd0ce0 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/86a8543910694c26a96e747052dd0ce0 2024-11-26T15:26:32,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/66f108a8eae44873800e8293790742f3 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/66f108a8eae44873800e8293790742f3 2024-11-26T15:26:32,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a1a8024dc39f43a380bcaa1b5c4b2861 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/a1a8024dc39f43a380bcaa1b5c4b2861 2024-11-26T15:26:32,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/aea17605460e471dad01e1665909ce6c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/aea17605460e471dad01e1665909ce6c 2024-11-26T15:26:32,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/84e4329c20014cafb4cd64d4c61ab2b3 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/84e4329c20014cafb4cd64d4c61ab2b3 2024-11-26T15:26:32,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/17af15ba911a45a99156eda1ce26f189 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/17af15ba911a45a99156eda1ce26f189 2024-11-26T15:26:32,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/1a63438ec1734329b92b2ba07d6bcc67 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/1a63438ec1734329b92b2ba07d6bcc67 2024-11-26T15:26:32,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/c20949bbe5c94b7e877a69b18e491dde to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/c20949bbe5c94b7e877a69b18e491dde 2024-11-26T15:26:32,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/1d71d7757bcd4b66a8038c66fa907842 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/1d71d7757bcd4b66a8038c66fa907842 2024-11-26T15:26:32,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/15f07c1c60f240d4ad084f8402568059 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/15f07c1c60f240d4ad084f8402568059 2024-11-26T15:26:32,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/addc074fd89f49579a6c3db982f3900f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/addc074fd89f49579a6c3db982f3900f 2024-11-26T15:26:32,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/512e8f8c64b94e4c9ea8838b9746a5b5 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/512e8f8c64b94e4c9ea8838b9746a5b5 2024-11-26T15:26:32,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/a51baa5b81a142f6ad75a2378272671b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/233b73eac05f49bca14e94f7a523feca, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/3b0f8b5bec654f0197675d2715e7d315, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/86489ac4509542ea9775026aaae5f392, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/a5b19dd2ca314b18abdc360ecfdc9e41, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/c6585a62282945af93b9f852c106330c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/5ef107c85ad34bbf92fa8078901d2373, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0c62ae1f11854adda942eba096a84392, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/fc29ea180e1f4a4ab6173ee9793176c8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/49d1cb55ca634bd98e7891d687420a20, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/33c07988b7ce4f2f88bcc04a58a45841, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/740104c616974171ac3fdd126d1b67af, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/3a5d2502359a47d7a8537b4b7fd2083a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/ec495b19db85421f9ae9701dc627592e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/b56d17db292549938bff7098a48b7df7, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0a0667cab4ef46cc91db0c8671e9828e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/1512d9a4d2e24ac4aa73507c4797465d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0d5aed8231a147aa82f81b0709af97f4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/c40bc4f2ec314f1693505f3967a791d2, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0dd72609ed8944c6b1ac7fd3f2d990a5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/3d83fa6a67b146169e5e938bba1c0586, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/2f35bc4208a548b0a99ae319a720b455, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/9c51359172394ac9b19fa6121d8ff5df, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/35077b8f380f48ceaa4638c572dbea36, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/844edb36fdaa4dec819ad656bda7717b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/bf9a252cedad4c34b79a5d2b4229c82c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/8aa068ee2f8641e0a061fa4cecd8a30d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/22b413650d5c4a4996f0337c18bb29ba] to archive 2024-11-26T15:26:32,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:26:32,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/a51baa5b81a142f6ad75a2378272671b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/a51baa5b81a142f6ad75a2378272671b 2024-11-26T15:26:32,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/233b73eac05f49bca14e94f7a523feca to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/233b73eac05f49bca14e94f7a523feca 2024-11-26T15:26:32,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/3b0f8b5bec654f0197675d2715e7d315 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/3b0f8b5bec654f0197675d2715e7d315 2024-11-26T15:26:32,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/86489ac4509542ea9775026aaae5f392 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/86489ac4509542ea9775026aaae5f392 2024-11-26T15:26:32,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/a5b19dd2ca314b18abdc360ecfdc9e41 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/a5b19dd2ca314b18abdc360ecfdc9e41 2024-11-26T15:26:32,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/c6585a62282945af93b9f852c106330c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/c6585a62282945af93b9f852c106330c 2024-11-26T15:26:32,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/5ef107c85ad34bbf92fa8078901d2373 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/5ef107c85ad34bbf92fa8078901d2373 2024-11-26T15:26:32,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0c62ae1f11854adda942eba096a84392 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0c62ae1f11854adda942eba096a84392 2024-11-26T15:26:32,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/fc29ea180e1f4a4ab6173ee9793176c8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/fc29ea180e1f4a4ab6173ee9793176c8 2024-11-26T15:26:32,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/49d1cb55ca634bd98e7891d687420a20 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/49d1cb55ca634bd98e7891d687420a20 2024-11-26T15:26:32,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/33c07988b7ce4f2f88bcc04a58a45841 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/33c07988b7ce4f2f88bcc04a58a45841 2024-11-26T15:26:32,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/740104c616974171ac3fdd126d1b67af to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/740104c616974171ac3fdd126d1b67af 2024-11-26T15:26:32,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/3a5d2502359a47d7a8537b4b7fd2083a to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/3a5d2502359a47d7a8537b4b7fd2083a 2024-11-26T15:26:32,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/ec495b19db85421f9ae9701dc627592e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/ec495b19db85421f9ae9701dc627592e 2024-11-26T15:26:32,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/b56d17db292549938bff7098a48b7df7 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/b56d17db292549938bff7098a48b7df7 2024-11-26T15:26:32,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0a0667cab4ef46cc91db0c8671e9828e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0a0667cab4ef46cc91db0c8671e9828e 2024-11-26T15:26:32,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/1512d9a4d2e24ac4aa73507c4797465d to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/1512d9a4d2e24ac4aa73507c4797465d 2024-11-26T15:26:32,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0d5aed8231a147aa82f81b0709af97f4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0d5aed8231a147aa82f81b0709af97f4 2024-11-26T15:26:32,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/c40bc4f2ec314f1693505f3967a791d2 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/c40bc4f2ec314f1693505f3967a791d2 2024-11-26T15:26:32,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0dd72609ed8944c6b1ac7fd3f2d990a5 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/0dd72609ed8944c6b1ac7fd3f2d990a5 2024-11-26T15:26:32,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/3d83fa6a67b146169e5e938bba1c0586 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/3d83fa6a67b146169e5e938bba1c0586 2024-11-26T15:26:32,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/2f35bc4208a548b0a99ae319a720b455 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/2f35bc4208a548b0a99ae319a720b455 2024-11-26T15:26:32,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/9c51359172394ac9b19fa6121d8ff5df to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/9c51359172394ac9b19fa6121d8ff5df 2024-11-26T15:26:32,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/35077b8f380f48ceaa4638c572dbea36 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/35077b8f380f48ceaa4638c572dbea36 2024-11-26T15:26:32,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/844edb36fdaa4dec819ad656bda7717b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/844edb36fdaa4dec819ad656bda7717b 2024-11-26T15:26:32,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/bf9a252cedad4c34b79a5d2b4229c82c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/bf9a252cedad4c34b79a5d2b4229c82c 2024-11-26T15:26:32,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/8aa068ee2f8641e0a061fa4cecd8a30d to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/8aa068ee2f8641e0a061fa4cecd8a30d 2024-11-26T15:26:32,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/22b413650d5c4a4996f0337c18bb29ba to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/22b413650d5c4a4996f0337c18bb29ba 2024-11-26T15:26:32,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/f9d27e5bab154c7aaca3f293a106560b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/cab214d9b343481cb1e69ec7ac585fa7, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/47d61a2f52ef4a3e8001890388dfa0b4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/daa86c5deb014f3f91d2b090461e8451, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/0432fcd906f64a4bb08e8dfee5b6aa74, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/4577f58b0eed4c9cbe63c11a184433fc, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/ebe9cef98bc14eee9118dcee66b38a42, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/bbbb25bda2ab4a139bfd45f5f89d57c5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/688bdbcf1a274204aafd6f539de3c168, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/9691a598f17d4c76841e14b0684eebfd, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/39b7d4f653504bac9f680f72bb058a99, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/95743450405e4b0987797d70e395d6cf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/c85d2f43b83244d3a799fbf4499d0bd4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/072d1c41c6be43db81678189dd004054, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/174a3ee57c2c4339972c255493b7da49, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/931ec6dd21d24b91b48c914c52a53ca4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/6f733a4c9f32410498ebd03a49a05b6f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/8affdca4b5c04f048f65922ff5c3841b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/677afdbc96344dbb97d38ac16a4f7a10, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/beaea8a402bf4dcb99b47f89c7057131, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/8b6355ab1dc346b38d93b52ba75a7bf7, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/15cc8348541d46268d518b2e9136c5b2, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/e7a75a9f61fa4adf8e05f11094fbc943, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/1253593379904d9198a9e41296124ac9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/61f4d8ce9f974459ba8446e3588fe5d9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/c3cb65ea7279459294fbc87ac8a30848, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/6d6a43ee35484434a77d1d0360ea857e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/27d6573e3e844b7fab822d33b3e24788] to archive 2024-11-26T15:26:32,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:26:32,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/f9d27e5bab154c7aaca3f293a106560b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/f9d27e5bab154c7aaca3f293a106560b 2024-11-26T15:26:32,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/cab214d9b343481cb1e69ec7ac585fa7 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/cab214d9b343481cb1e69ec7ac585fa7 2024-11-26T15:26:32,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/47d61a2f52ef4a3e8001890388dfa0b4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/47d61a2f52ef4a3e8001890388dfa0b4 2024-11-26T15:26:32,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/daa86c5deb014f3f91d2b090461e8451 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/daa86c5deb014f3f91d2b090461e8451 2024-11-26T15:26:32,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/0432fcd906f64a4bb08e8dfee5b6aa74 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/0432fcd906f64a4bb08e8dfee5b6aa74 2024-11-26T15:26:32,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/4577f58b0eed4c9cbe63c11a184433fc to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/4577f58b0eed4c9cbe63c11a184433fc 2024-11-26T15:26:32,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/ebe9cef98bc14eee9118dcee66b38a42 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/ebe9cef98bc14eee9118dcee66b38a42 2024-11-26T15:26:32,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/bbbb25bda2ab4a139bfd45f5f89d57c5 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/bbbb25bda2ab4a139bfd45f5f89d57c5 2024-11-26T15:26:32,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/688bdbcf1a274204aafd6f539de3c168 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/688bdbcf1a274204aafd6f539de3c168 2024-11-26T15:26:32,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/9691a598f17d4c76841e14b0684eebfd to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/9691a598f17d4c76841e14b0684eebfd 2024-11-26T15:26:32,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/39b7d4f653504bac9f680f72bb058a99 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/39b7d4f653504bac9f680f72bb058a99 2024-11-26T15:26:32,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/95743450405e4b0987797d70e395d6cf to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/95743450405e4b0987797d70e395d6cf 2024-11-26T15:26:32,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/c85d2f43b83244d3a799fbf4499d0bd4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/c85d2f43b83244d3a799fbf4499d0bd4 2024-11-26T15:26:32,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/072d1c41c6be43db81678189dd004054 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/072d1c41c6be43db81678189dd004054 2024-11-26T15:26:32,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/174a3ee57c2c4339972c255493b7da49 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/174a3ee57c2c4339972c255493b7da49 2024-11-26T15:26:32,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/931ec6dd21d24b91b48c914c52a53ca4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/931ec6dd21d24b91b48c914c52a53ca4 2024-11-26T15:26:32,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/6f733a4c9f32410498ebd03a49a05b6f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/6f733a4c9f32410498ebd03a49a05b6f 2024-11-26T15:26:32,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/8affdca4b5c04f048f65922ff5c3841b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/8affdca4b5c04f048f65922ff5c3841b 2024-11-26T15:26:32,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/677afdbc96344dbb97d38ac16a4f7a10 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/677afdbc96344dbb97d38ac16a4f7a10 2024-11-26T15:26:32,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/beaea8a402bf4dcb99b47f89c7057131 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/beaea8a402bf4dcb99b47f89c7057131 2024-11-26T15:26:32,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/8b6355ab1dc346b38d93b52ba75a7bf7 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/8b6355ab1dc346b38d93b52ba75a7bf7 2024-11-26T15:26:32,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/15cc8348541d46268d518b2e9136c5b2 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/15cc8348541d46268d518b2e9136c5b2 2024-11-26T15:26:32,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/e7a75a9f61fa4adf8e05f11094fbc943 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/e7a75a9f61fa4adf8e05f11094fbc943 2024-11-26T15:26:32,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/1253593379904d9198a9e41296124ac9 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/1253593379904d9198a9e41296124ac9 2024-11-26T15:26:32,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/61f4d8ce9f974459ba8446e3588fe5d9 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/61f4d8ce9f974459ba8446e3588fe5d9 2024-11-26T15:26:32,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/c3cb65ea7279459294fbc87ac8a30848 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/c3cb65ea7279459294fbc87ac8a30848 2024-11-26T15:26:32,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/6d6a43ee35484434a77d1d0360ea857e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/6d6a43ee35484434a77d1d0360ea857e 2024-11-26T15:26:32,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/27d6573e3e844b7fab822d33b3e24788 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/27d6573e3e844b7fab822d33b3e24788 2024-11-26T15:26:32,101 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/recovered.edits/461.seqid, newMaxSeqId=461, maxSeqId=1 2024-11-26T15:26:32,102 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. 2024-11-26T15:26:32,102 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 21fb246aea47afdec08e90541f5d61cc: Waiting for close lock at 1732634790752Running coprocessor pre-close hooks at 1732634790752Disabling compacts and flushes for region at 1732634790752Disabling writes for close at 1732634790752Obtaining lock to block concurrent updates at 1732634790753 (+1 ms)Preparing flush snapshotting stores in 21fb246aea47afdec08e90541f5d61cc at 1732634790753Finished memstore snapshotting TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732634790753Flushing stores of TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc. at 1732634790754 (+1 ms)Flushing 21fb246aea47afdec08e90541f5d61cc/A: creating writer at 1732634790755 (+1 ms)Flushing 21fb246aea47afdec08e90541f5d61cc/A: appending metadata at 1732634790760 (+5 ms)Flushing 21fb246aea47afdec08e90541f5d61cc/A: closing flushed file at 1732634790760Flushing 21fb246aea47afdec08e90541f5d61cc/B: creating writer at 1732634791176 (+416 ms)Flushing 21fb246aea47afdec08e90541f5d61cc/B: appending metadata at 1732634791179 (+3 ms)Flushing 21fb246aea47afdec08e90541f5d61cc/B: closing flushed file at 1732634791179Flushing 21fb246aea47afdec08e90541f5d61cc/C: creating writer at 1732634791592 (+413 ms)Flushing 21fb246aea47afdec08e90541f5d61cc/C: appending metadata at 1732634791596 (+4 ms)Flushing 21fb246aea47afdec08e90541f5d61cc/C: closing flushed file at 1732634791596Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3518e2bd: reopening flushed file at 1732634792007 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@666778cf: reopening flushed file at 1732634792012 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7086c023: reopening flushed file at 1732634792017 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 21fb246aea47afdec08e90541f5d61cc in 1269ms, sequenceid=458, compaction requested=true at 1732634792021 (+4 ms)Writing region close event to WAL at 1732634792098 (+77 ms)Running coprocessor post-close hooks at 1732634792102 (+4 ms)Closed at 1732634792102 2024-11-26T15:26:32,103 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:32,104 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=21fb246aea47afdec08e90541f5d61cc, regionState=CLOSED 2024-11-26T15:26:32,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:26:32,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-26T15:26:32,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 21fb246aea47afdec08e90541f5d61cc, server=6cc2bc8bf733,45729,1732634697146 in 1.5080 sec 2024-11-26T15:26:32,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-26T15:26:32,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21fb246aea47afdec08e90541f5d61cc, UNASSIGN in 1.5130 sec 2024-11-26T15:26:32,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-26T15:26:32,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5170 sec 2024-11-26T15:26:32,111 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634792111"}]},"ts":"1732634792111"} 2024-11-26T15:26:32,113 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T15:26:32,113 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T15:26:32,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5270 sec 2024-11-26T15:26:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-26T15:26:32,733 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:32,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T15:26:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:26:32,736 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:26:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-26T15:26:32,737 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:26:32,739 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:32,743 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/recovered.edits] 2024-11-26T15:26:32,746 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/58db72b8a0bc423696ba82bb6d05d97e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/58db72b8a0bc423696ba82bb6d05d97e 2024-11-26T15:26:32,747 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/9e71c16b0edf4cfab800dce4f7d70f1c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/9e71c16b0edf4cfab800dce4f7d70f1c 2024-11-26T15:26:32,747 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/bf2590d9b69c45ef82ecd976be2c8066 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/bf2590d9b69c45ef82ecd976be2c8066 2024-11-26T15:26:32,748 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/f1cf0b48a1664765998f6847b35c3230 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/A/f1cf0b48a1664765998f6847b35c3230 2024-11-26T15:26:32,750 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/571ebf6541b444c5967d581e73425a54 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/571ebf6541b444c5967d581e73425a54 2024-11-26T15:26:32,751 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/7ceece3900f343e88bff767b27f24cee to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/7ceece3900f343e88bff767b27f24cee 2024-11-26T15:26:32,752 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/bc6a67602a0345fc929e0d8ee4b1134e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/bc6a67602a0345fc929e0d8ee4b1134e 2024-11-26T15:26:32,753 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/d43e98b34d7d4997aae4032f724118de to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/B/d43e98b34d7d4997aae4032f724118de 2024-11-26T15:26:32,755 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/77efa1bfac6b4552bdf10591cd59d512 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/77efa1bfac6b4552bdf10591cd59d512 2024-11-26T15:26:32,756 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/c950b24e685143349076ee38bfcddb9f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/c950b24e685143349076ee38bfcddb9f 2024-11-26T15:26:32,757 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/cb7680ce7f134672b12da3969eb112a0 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/cb7680ce7f134672b12da3969eb112a0 2024-11-26T15:26:32,758 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/e00a0b1d2dc8466280ed565bf391ac0c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/C/e00a0b1d2dc8466280ed565bf391ac0c 2024-11-26T15:26:32,761 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/recovered.edits/461.seqid to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc/recovered.edits/461.seqid 2024-11-26T15:26:32,762 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/21fb246aea47afdec08e90541f5d61cc 2024-11-26T15:26:32,762 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T15:26:32,765 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:26:32,769 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T15:26:32,772 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T15:26:32,773 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:26:32,773 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T15:26:32,774 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732634792773"}]},"ts":"9223372036854775807"} 2024-11-26T15:26:32,776 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T15:26:32,776 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 21fb246aea47afdec08e90541f5d61cc, NAME => 'TestAcidGuarantees,,1732634759961.21fb246aea47afdec08e90541f5d61cc.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T15:26:32,776 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T15:26:32,776 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732634792776"}]},"ts":"9223372036854775807"} 2024-11-26T15:26:32,778 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T15:26:32,779 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:26:32,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-26T15:26:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-26T15:26:32,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T15:26:32,843 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:32,858 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=236 (was 238), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=473 (was 590), ProcessCount=11 (was 11), AvailableMemoryMB=9226 (was 9163) - AvailableMemoryMB LEAK? - 2024-11-26T15:26:32,868 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=236, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=473, ProcessCount=11, AvailableMemoryMB=9225 2024-11-26T15:26:32,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T15:26:32,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T15:26:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T15:26:32,872 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T15:26:32,872 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:32,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-26T15:26:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T15:26:32,873 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T15:26:32,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742165_1341 (size=959) 2024-11-26T15:26:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T15:26:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T15:26:33,286 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6 2024-11-26T15:26:33,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742166_1342 (size=53) 2024-11-26T15:26:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T15:26:33,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:26:33,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ef027b3b2a337d7b7f5ab878c67d25f8, disabling compactions & flushes 2024-11-26T15:26:33,698 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:33,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:33,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. after waiting 0 ms 2024-11-26T15:26:33,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:33,698 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:33,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ef027b3b2a337d7b7f5ab878c67d25f8: Waiting for close lock at 1732634793698Disabling compacts and flushes for region at 1732634793698Disabling writes for close at 1732634793698Writing region close event to WAL at 1732634793698Closed at 1732634793698 2024-11-26T15:26:33,700 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T15:26:33,701 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732634793700"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732634793700"}]},"ts":"1732634793700"} 2024-11-26T15:26:33,703 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T15:26:33,705 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T15:26:33,705 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634793705"}]},"ts":"1732634793705"} 2024-11-26T15:26:33,707 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T15:26:33,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef027b3b2a337d7b7f5ab878c67d25f8, ASSIGN}] 2024-11-26T15:26:33,708 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef027b3b2a337d7b7f5ab878c67d25f8, ASSIGN 2024-11-26T15:26:33,709 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef027b3b2a337d7b7f5ab878c67d25f8, ASSIGN; state=OFFLINE, location=6cc2bc8bf733,45729,1732634697146; forceNewPlan=false, retain=false 2024-11-26T15:26:33,861 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=ef027b3b2a337d7b7f5ab878c67d25f8, regionState=OPENING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:33,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef027b3b2a337d7b7f5ab878c67d25f8, ASSIGN because future has completed 2024-11-26T15:26:33,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:26:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T15:26:34,025 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:34,025 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:26:34,026 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:34,026 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:26:34,026 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:34,027 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:34,029 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:34,031 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef027b3b2a337d7b7f5ab878c67d25f8 columnFamilyName A 2024-11-26T15:26:34,032 DEBUG [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:34,032 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] regionserver.HStore(327): Store=ef027b3b2a337d7b7f5ab878c67d25f8/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:26:34,032 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:34,034 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef027b3b2a337d7b7f5ab878c67d25f8 columnFamilyName B 2024-11-26T15:26:34,034 DEBUG [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:34,034 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] regionserver.HStore(327): Store=ef027b3b2a337d7b7f5ab878c67d25f8/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:26:34,035 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:34,036 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef027b3b2a337d7b7f5ab878c67d25f8 columnFamilyName C 2024-11-26T15:26:34,036 DEBUG [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:34,037 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] regionserver.HStore(327): Store=ef027b3b2a337d7b7f5ab878c67d25f8/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:26:34,037 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:34,038 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:34,038 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:34,039 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:34,039 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:34,040 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T15:26:34,040 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:34,042 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:26:34,042 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened ef027b3b2a337d7b7f5ab878c67d25f8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59537038, jitterRate=-0.11282899975776672}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T15:26:34,042 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:34,043 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for ef027b3b2a337d7b7f5ab878c67d25f8: Running coprocessor pre-open hook at 1732634794027Writing region info on filesystem at 1732634794027Initializing all the Stores at 1732634794028 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634794028Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634794029 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634794029Cleaning up temporary data from old regions at 1732634794039 (+10 ms)Running coprocessor post-open hooks at 1732634794042 (+3 ms)Region opened successfully at 1732634794042 2024-11-26T15:26:34,043 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., pid=89, masterSystemTime=1732634794019 2024-11-26T15:26:34,044 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:34,044 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:34,045 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=ef027b3b2a337d7b7f5ab878c67d25f8, regionState=OPEN, openSeqNum=2, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:34,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:26:34,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-26T15:26:34,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 in 181 msec 2024-11-26T15:26:34,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-26T15:26:34,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef027b3b2a337d7b7f5ab878c67d25f8, ASSIGN in 341 msec 2024-11-26T15:26:34,051 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T15:26:34,051 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634794051"}]},"ts":"1732634794051"} 2024-11-26T15:26:34,052 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T15:26:34,053 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T15:26:34,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-11-26T15:26:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-26T15:26:35,024 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:35,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a99cad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:35,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:35,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:35,030 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:35,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:35,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:35,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d4c1eb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:35,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:35,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:35,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:35,034 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62cf9f55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:35,036 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:35,037 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:35,037 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T15:26:35,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 6cc2bc8bf733,37081,1732634696319 2024-11-26T15:26:35,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@38adb4e8 2024-11-26T15:26:35,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T15:26:35,039 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T15:26:35,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T15:26:35,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T15:26:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T15:26:35,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742167_1343 (size=995) 2024-11-26T15:26:35,454 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-26T15:26:35,454 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-26T15:26:35,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T15:26:35,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef027b3b2a337d7b7f5ab878c67d25f8, REOPEN/MOVE}] 2024-11-26T15:26:35,463 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef027b3b2a337d7b7f5ab878c67d25f8, REOPEN/MOVE 2024-11-26T15:26:35,463 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=ef027b3b2a337d7b7f5ab878c67d25f8, regionState=CLOSING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:35,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef027b3b2a337d7b7f5ab878c67d25f8, REOPEN/MOVE because future has completed 2024-11-26T15:26:35,465 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T15:26:35,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:26:35,617 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:35,618 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T15:26:35,618 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing ef027b3b2a337d7b7f5ab878c67d25f8, disabling compactions & flushes 2024-11-26T15:26:35,618 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:35,618 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:35,618 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. after waiting 0 ms 2024-11-26T15:26:35,618 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:35,625 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T15:26:35,626 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:35,626 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for ef027b3b2a337d7b7f5ab878c67d25f8: Waiting for close lock at 1732634795618Running coprocessor pre-close hooks at 1732634795618Disabling compacts and flushes for region at 1732634795618Disabling writes for close at 1732634795618Writing region close event to WAL at 1732634795619 (+1 ms)Running coprocessor post-close hooks at 1732634795626 (+7 ms)Closed at 1732634795626 2024-11-26T15:26:35,626 WARN [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: ef027b3b2a337d7b7f5ab878c67d25f8 to self. 2024-11-26T15:26:35,629 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:35,630 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=ef027b3b2a337d7b7f5ab878c67d25f8, regionState=CLOSED 2024-11-26T15:26:35,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:26:35,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-26T15:26:35,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 in 169 msec 2024-11-26T15:26:35,636 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef027b3b2a337d7b7f5ab878c67d25f8, REOPEN/MOVE; state=CLOSED, location=6cc2bc8bf733,45729,1732634697146; forceNewPlan=false, retain=true 2024-11-26T15:26:35,787 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=ef027b3b2a337d7b7f5ab878c67d25f8, regionState=OPENING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:35,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef027b3b2a337d7b7f5ab878c67d25f8, REOPEN/MOVE because future has completed 2024-11-26T15:26:35,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:26:35,950 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:35,951 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:26:35,951 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:35,951 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:26:35,951 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:35,951 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:35,954 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:35,955 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef027b3b2a337d7b7f5ab878c67d25f8 columnFamilyName A 2024-11-26T15:26:35,956 DEBUG [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:35,957 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] regionserver.HStore(327): Store=ef027b3b2a337d7b7f5ab878c67d25f8/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:26:35,958 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:35,959 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef027b3b2a337d7b7f5ab878c67d25f8 columnFamilyName B 2024-11-26T15:26:35,959 DEBUG [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:35,960 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] regionserver.HStore(327): Store=ef027b3b2a337d7b7f5ab878c67d25f8/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:26:35,960 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:35,961 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef027b3b2a337d7b7f5ab878c67d25f8 columnFamilyName C 2024-11-26T15:26:35,961 DEBUG [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:35,961 INFO [StoreOpener-ef027b3b2a337d7b7f5ab878c67d25f8-1 {}] regionserver.HStore(327): Store=ef027b3b2a337d7b7f5ab878c67d25f8/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:26:35,962 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:35,963 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:35,964 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:35,965 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:35,965 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:35,965 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T15:26:35,967 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:35,968 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened ef027b3b2a337d7b7f5ab878c67d25f8; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64267484, jitterRate=-0.042339861392974854}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T15:26:35,968 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:35,968 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for ef027b3b2a337d7b7f5ab878c67d25f8: Running coprocessor pre-open hook at 1732634795952Writing region info on filesystem at 1732634795952Initializing all the Stores at 1732634795953 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634795953Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634795953Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634795953Cleaning up temporary data from old regions at 1732634795965 (+12 ms)Running coprocessor post-open hooks at 1732634795968 (+3 ms)Region opened successfully at 1732634795968 2024-11-26T15:26:35,969 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., pid=94, masterSystemTime=1732634795945 2024-11-26T15:26:35,971 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:35,971 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:35,971 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=ef027b3b2a337d7b7f5ab878c67d25f8, regionState=OPEN, openSeqNum=5, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:35,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:26:35,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-26T15:26:35,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 in 183 msec 2024-11-26T15:26:35,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-26T15:26:35,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef027b3b2a337d7b7f5ab878c67d25f8, REOPEN/MOVE in 514 msec 2024-11-26T15:26:35,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-26T15:26:35,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 520 msec 2024-11-26T15:26:35,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 943 msec 2024-11-26T15:26:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-26T15:26:35,986 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:35,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff492b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:35,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:35,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:35,989 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3197d5fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:35,991 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fcc5b1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:35,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c07883b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:35,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:35,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:35,993 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10447488, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:35,995 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:35,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a4911a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:35,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d62a3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:35,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:35,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:35,997 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f3dda43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:35,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:35,998 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fe12bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:35,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6513d18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:35,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:35,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:36,000 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:36,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:36,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:36,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28508b7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:36,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:36,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:36,002 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77933dc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510f8e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,003 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:36,003 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:36,003 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e04b85f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:36,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:36,005 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:36,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@549ad260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d0f7be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:36,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:36,007 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128a453c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:36,008 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:36,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f5facf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a06cc9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,009 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:36,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:36,009 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:36,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:36,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@447d1f8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:36,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:36,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:36,010 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46162ebb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5990cd3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,011 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:36,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:36,012 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63760f94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:36,013 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:36,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bf5a608, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8367c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:36,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:36,015 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:36,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:36,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:36,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61e942b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:36,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:36,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:36,016 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:36,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63336bcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df1ab09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:26:36,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:26:36,019 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:26:36,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:26:36,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:26:36,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7afd9247, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:26:36,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:26:36,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:36,020 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:26:36,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a43b7d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:26:36,021 DEBUG [Thread-1562 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:36,021 DEBUG [Thread-1560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:36,021 DEBUG [Thread-1575 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:36,022 DEBUG [Thread-1564 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:36,022 DEBUG [Thread-1577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:36,022 DEBUG [Thread-1566 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:36,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:36,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:36,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:36,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:36,023 DEBUG [Thread-1568 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:36,023 DEBUG [Thread-1579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:36,023 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:36,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:36,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:36,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:36,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:36,024 DEBUG [Thread-1571 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:36,024 DEBUG [Thread-1573 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:26:36,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:36,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:36,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:36,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-26T15:26:36,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:36,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:36,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T15:26:36,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:36,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:26:36,027 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:36,027 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:36,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:26:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:26:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:26:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:26:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:26:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:26:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:26:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:26:36,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:36,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:26:36,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:26:36,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:26:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:26:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:36,036 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:36,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T15:26:36,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126be14611b9280449685aaa584272a21b1_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634796036/Put/seqid=0 2024-11-26T15:26:36,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742168_1344 (size=12154) 2024-11-26T15:26:36,071 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:36,075 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126be14611b9280449685aaa584272a21b1_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126be14611b9280449685aaa584272a21b1_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:36,076 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/5c0a2828eb334c8894434a262680d3a9, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:36,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/5c0a2828eb334c8894434a262680d3a9 is 175, key is test_row_0/A:col10/1732634796036/Put/seqid=0 2024-11-26T15:26:36,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742169_1345 (size=30955) 2024-11-26T15:26:36,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634856073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634856075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634856075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634856076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634856081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T15:26:36,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T15:26:36,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:36,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:36,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:36,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:36,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:36,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634856182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634856192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634856192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634856192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634856192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T15:26:36,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:36,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:36,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:36,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:36,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:36,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T15:26:36,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634856392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634856402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634856402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634856402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634856402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,479 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/5c0a2828eb334c8894434a262680d3a9 2024-11-26T15:26:36,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T15:26:36,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:36,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:36,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:36,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:36,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:36,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/267854020ddc461bae9382b52ba088dd is 50, key is test_row_0/B:col10/1732634796036/Put/seqid=0 2024-11-26T15:26:36,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742170_1346 (size=12001) 2024-11-26T15:26:36,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/267854020ddc461bae9382b52ba088dd 2024-11-26T15:26:36,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/9e89209d90a04467a2cfa4ccf516bb29 is 50, key is test_row_0/C:col10/1732634796036/Put/seqid=0 2024-11-26T15:26:36,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742171_1347 (size=12001) 2024-11-26T15:26:36,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T15:26:36,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:36,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:36,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:36,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:36,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T15:26:36,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634856702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634856712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634856722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634856722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634856722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:36,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T15:26:36,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:36,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:36,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:36,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:36,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:36,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/9e89209d90a04467a2cfa4ccf516bb29 2024-11-26T15:26:36,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/5c0a2828eb334c8894434a262680d3a9 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/5c0a2828eb334c8894434a262680d3a9 2024-11-26T15:26:36,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/5c0a2828eb334c8894434a262680d3a9, entries=150, sequenceid=16, filesize=30.2 K 2024-11-26T15:26:36,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/267854020ddc461bae9382b52ba088dd as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/267854020ddc461bae9382b52ba088dd 2024-11-26T15:26:36,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/267854020ddc461bae9382b52ba088dd, entries=150, sequenceid=16, filesize=11.7 K 2024-11-26T15:26:36,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/9e89209d90a04467a2cfa4ccf516bb29 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/9e89209d90a04467a2cfa4ccf516bb29 2024-11-26T15:26:36,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/9e89209d90a04467a2cfa4ccf516bb29, entries=150, sequenceid=16, filesize=11.7 K 2024-11-26T15:26:36,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ef027b3b2a337d7b7f5ab878c67d25f8 in 905ms, sequenceid=16, compaction requested=false 2024-11-26T15:26:36,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:36,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T15:26:36,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:36,943 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T15:26:36,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e0010603f7404b3f8fb76547ca167a1e_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634796071/Put/seqid=0 2024-11-26T15:26:36,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742172_1348 (size=12154) 2024-11-26T15:26:36,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:36,966 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e0010603f7404b3f8fb76547ca167a1e_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e0010603f7404b3f8fb76547ca167a1e_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:36,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/a27c1eefeddf4520aa0420183801682c, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:36,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/a27c1eefeddf4520aa0420183801682c is 175, key is test_row_0/A:col10/1732634796071/Put/seqid=0 2024-11-26T15:26:36,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742173_1349 (size=30955) 2024-11-26T15:26:36,988 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/a27c1eefeddf4520aa0420183801682c 2024-11-26T15:26:36,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/96be9cabf86b446282c524b3e8b9b2e5 is 50, key is test_row_0/B:col10/1732634796071/Put/seqid=0 2024-11-26T15:26:37,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742174_1350 (size=12001) 2024-11-26T15:26:37,005 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/96be9cabf86b446282c524b3e8b9b2e5 2024-11-26T15:26:37,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/821d7cdd62d2474183358ab803b785be is 50, key is test_row_0/C:col10/1732634796071/Put/seqid=0 2024-11-26T15:26:37,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742175_1351 (size=12001) 2024-11-26T15:26:37,023 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/821d7cdd62d2474183358ab803b785be 2024-11-26T15:26:37,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/a27c1eefeddf4520aa0420183801682c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a27c1eefeddf4520aa0420183801682c 2024-11-26T15:26:37,034 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a27c1eefeddf4520aa0420183801682c, entries=150, sequenceid=40, filesize=30.2 K 2024-11-26T15:26:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/96be9cabf86b446282c524b3e8b9b2e5 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/96be9cabf86b446282c524b3e8b9b2e5 2024-11-26T15:26:37,039 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/96be9cabf86b446282c524b3e8b9b2e5, entries=150, sequenceid=40, filesize=11.7 K 2024-11-26T15:26:37,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/821d7cdd62d2474183358ab803b785be as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/821d7cdd62d2474183358ab803b785be 2024-11-26T15:26:37,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,044 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/821d7cdd62d2474183358ab803b785be, entries=150, sequenceid=40, filesize=11.7 K 2024-11-26T15:26:37,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,045 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for ef027b3b2a337d7b7f5ab878c67d25f8 in 102ms, sequenceid=40, compaction requested=false 2024-11-26T15:26:37,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:37,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:37,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-26T15:26:37,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-26T15:26:37,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,048 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-26T15:26:37,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,048 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0190 sec 2024-11-26T15:26:37,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.0250 sec 2024-11-26T15:26:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T15:26:37,164 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:37,165 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-26T15:26:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T15:26:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,167 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,168 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T15:26:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T15:26:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:37,321 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-26T15:26:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:37,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c4b6583316d1477daac6cf4a035757ff_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634797307/Put/seqid=0 2024-11-26T15:26:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742176_1352 (size=19474) 2024-11-26T15:26:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:37,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634857375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634857374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:37,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634857388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:37,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634857390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:37,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634857391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T15:26:37,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634857503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:37,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634857503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:37,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634857503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634857503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634857503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:37,647 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T15:26:37,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634857712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:37,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634857712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:37,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634857712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:37,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634857712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634857712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:37,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:37,753 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c4b6583316d1477daac6cf4a035757ff_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c4b6583316d1477daac6cf4a035757ff_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:37,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/bca6d2aaac5b44b7b2d52e331261b0fd, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:37,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/bca6d2aaac5b44b7b2d52e331261b0fd is 175, key is test_row_0/A:col10/1732634797307/Put/seqid=0 2024-11-26T15:26:37,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742177_1353 (size=56733) 2024-11-26T15:26:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T15:26:38,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634858023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:38,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:38,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634858023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:38,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:38,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634858023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:38,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634858023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:38,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634858023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:38,158 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/bca6d2aaac5b44b7b2d52e331261b0fd 2024-11-26T15:26:38,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/80ab99748e6e47d4a4dc03c862f3d6d9 is 50, key is test_row_0/B:col10/1732634797307/Put/seqid=0 2024-11-26T15:26:38,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742178_1354 (size=12001) 2024-11-26T15:26:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T15:26:38,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634858532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:38,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634858532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:38,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634858532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:38,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634858532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:38,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634858543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:38,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:38,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:38,581 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/80ab99748e6e47d4a4dc03c862f3d6d9 2024-11-26T15:26:38,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/89bdfe3150a1400f83e0485025580a5a is 50, key is test_row_0/C:col10/1732634797307/Put/seqid=0 2024-11-26T15:26:38,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742179_1355 (size=12001) 2024-11-26T15:26:39,003 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/89bdfe3150a1400f83e0485025580a5a 2024-11-26T15:26:39,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/bca6d2aaac5b44b7b2d52e331261b0fd as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/bca6d2aaac5b44b7b2d52e331261b0fd 2024-11-26T15:26:39,013 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/bca6d2aaac5b44b7b2d52e331261b0fd, entries=300, sequenceid=49, filesize=55.4 K 2024-11-26T15:26:39,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/80ab99748e6e47d4a4dc03c862f3d6d9 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/80ab99748e6e47d4a4dc03c862f3d6d9 2024-11-26T15:26:39,019 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/80ab99748e6e47d4a4dc03c862f3d6d9, entries=150, sequenceid=49, filesize=11.7 K 2024-11-26T15:26:39,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/89bdfe3150a1400f83e0485025580a5a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/89bdfe3150a1400f83e0485025580a5a 2024-11-26T15:26:39,028 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/89bdfe3150a1400f83e0485025580a5a, entries=150, sequenceid=49, filesize=11.7 K 2024-11-26T15:26:39,030 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for ef027b3b2a337d7b7f5ab878c67d25f8 in 1708ms, sequenceid=49, compaction requested=true 2024-11-26T15:26:39,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:39,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:39,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-26T15:26:39,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-26T15:26:39,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-26T15:26:39,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8660 sec 2024-11-26T15:26:39,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8710 sec 2024-11-26T15:26:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T15:26:39,313 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:39,314 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-26T15:26:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T15:26:39,317 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:39,318 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:39,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T15:26:39,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T15:26:39,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:39,471 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-26T15:26:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f96642ce4b5348b8aacc514674e27208_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634797388/Put/seqid=0 2024-11-26T15:26:39,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742180_1356 (size=12154) 2024-11-26T15:26:39,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:39,520 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f96642ce4b5348b8aacc514674e27208_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f96642ce4b5348b8aacc514674e27208_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:39,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/73fbf91aa56f47c997a5106e7c45ae04, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:39,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/73fbf91aa56f47c997a5106e7c45ae04 is 175, key is test_row_0/A:col10/1732634797388/Put/seqid=0 2024-11-26T15:26:39,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742181_1357 (size=30955) 2024-11-26T15:26:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:39,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:39,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634859552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:39,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634859552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:39,566 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in ef027b3b2a337d7b7f5ab878c67d25f8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:39,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:39,567 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in ef027b3b2a337d7b7f5ab878c67d25f8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:39,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:39,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634859564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:39,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634859566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:39,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634859564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T15:26:39,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634859683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:39,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634859683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:39,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634859692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:39,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:39,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634859902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634859902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:39,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:39,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:39,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634859913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:39,937 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/73fbf91aa56f47c997a5106e7c45ae04 2024-11-26T15:26:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T15:26:39,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/1b3f45d976834053bb8808220b78cd2b is 50, key is test_row_0/B:col10/1732634797388/Put/seqid=0 2024-11-26T15:26:39,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742182_1358 (size=12001) 2024-11-26T15:26:40,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634860223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:40,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634860224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:40,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:40,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634860233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:40,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:40,394 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/1b3f45d976834053bb8808220b78cd2b 2024-11-26T15:26:40,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/e8a56155e2ba4296b70267e5ceb42347 is 50, key is test_row_0/C:col10/1732634797388/Put/seqid=0 2024-11-26T15:26:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T15:26:40,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742183_1359 (size=12001) 2024-11-26T15:26:40,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:40,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634860732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:40,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634860742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:40,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:40,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634860753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:40,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:40,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:40,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:40,864 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/e8a56155e2ba4296b70267e5ceb42347 2024-11-26T15:26:40,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/73fbf91aa56f47c997a5106e7c45ae04 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/73fbf91aa56f47c997a5106e7c45ae04 2024-11-26T15:26:40,881 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/73fbf91aa56f47c997a5106e7c45ae04, entries=150, sequenceid=76, filesize=30.2 K 2024-11-26T15:26:40,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/1b3f45d976834053bb8808220b78cd2b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/1b3f45d976834053bb8808220b78cd2b 2024-11-26T15:26:40,889 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/1b3f45d976834053bb8808220b78cd2b, entries=150, sequenceid=76, filesize=11.7 K 2024-11-26T15:26:40,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/e8a56155e2ba4296b70267e5ceb42347 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/e8a56155e2ba4296b70267e5ceb42347 2024-11-26T15:26:40,896 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/e8a56155e2ba4296b70267e5ceb42347, entries=150, sequenceid=76, filesize=11.7 K 2024-11-26T15:26:40,900 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for ef027b3b2a337d7b7f5ab878c67d25f8 in 1429ms, sequenceid=76, compaction requested=true 2024-11-26T15:26:40,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:40,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:40,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-26T15:26:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-26T15:26:40,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-26T15:26:40,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5840 sec 2024-11-26T15:26:40,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.5910 sec 2024-11-26T15:26:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T15:26:41,463 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:41,463 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-26T15:26:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T15:26:41,465 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:41,466 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:41,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T15:26:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:41,593 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:41,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T15:26:41,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b9808e6b8e2b4e9cbb5a61d115b9879b_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634801591/Put/seqid=0 2024-11-26T15:26:41,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T15:26:41,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:41,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:41,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:41,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:41,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:41,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742184_1360 (size=14594) 2024-11-26T15:26:41,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634861737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:41,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634861748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:41,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634861763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:41,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634861763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:41,774 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in ef027b3b2a337d7b7f5ab878c67d25f8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:41,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:41,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:41,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:41,775 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in ef027b3b2a337d7b7f5ab878c67d25f8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:41,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:41,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:41,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:41,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T15:26:41,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:41,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:41,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:41,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:41,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T15:26:41,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634861772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:41,784 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in ef027b3b2a337d7b7f5ab878c67d25f8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:41,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634861862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:41,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634861862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:41,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T15:26:41,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:41,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:41,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:41,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:41,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,068 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:42,071 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b9808e6b8e2b4e9cbb5a61d115b9879b_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b9808e6b8e2b4e9cbb5a61d115b9879b_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:42,072 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/9b52c9092a564b5ea10c92a7347f890f, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:42,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/9b52c9092a564b5ea10c92a7347f890f is 175, key is test_row_0/A:col10/1732634801591/Put/seqid=0 2024-11-26T15:26:42,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742185_1361 (size=39549) 2024-11-26T15:26:42,077 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=87, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/9b52c9092a564b5ea10c92a7347f890f 2024-11-26T15:26:42,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634862073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:42,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T15:26:42,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:42,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:42,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:42,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/d7a8222c250442ffb7e26300a2eca345 is 50, key is test_row_0/B:col10/1732634801591/Put/seqid=0 2024-11-26T15:26:42,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742186_1362 (size=12001) 2024-11-26T15:26:42,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634862083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T15:26:42,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T15:26:42,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:42,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:42,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:42,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T15:26:42,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:42,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:42,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:42,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634862392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:42,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634862402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:42,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/d7a8222c250442ffb7e26300a2eca345 2024-11-26T15:26:42,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/3062de8224454b49846c0df66b0a27ca is 50, key is test_row_0/C:col10/1732634801591/Put/seqid=0 2024-11-26T15:26:42,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742187_1363 (size=12001) 2024-11-26T15:26:42,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T15:26:42,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:42,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:42,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:42,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T15:26:42,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T15:26:42,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:42,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:42,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:42,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T15:26:42,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:42,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:42,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:42,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:42,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/3062de8224454b49846c0df66b0a27ca 2024-11-26T15:26:42,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/9b52c9092a564b5ea10c92a7347f890f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/9b52c9092a564b5ea10c92a7347f890f 2024-11-26T15:26:42,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/9b52c9092a564b5ea10c92a7347f890f, entries=200, sequenceid=87, filesize=38.6 K 2024-11-26T15:26:42,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/d7a8222c250442ffb7e26300a2eca345 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/d7a8222c250442ffb7e26300a2eca345 2024-11-26T15:26:42,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/d7a8222c250442ffb7e26300a2eca345, entries=150, sequenceid=87, filesize=11.7 K 2024-11-26T15:26:42,912 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-26T15:26:42,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634862902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:42,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/3062de8224454b49846c0df66b0a27ca as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3062de8224454b49846c0df66b0a27ca 2024-11-26T15:26:42,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3062de8224454b49846c0df66b0a27ca, entries=150, sequenceid=87, filesize=11.7 K 2024-11-26T15:26:42,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634862913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:42,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:42,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:42,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:42,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ef027b3b2a337d7b7f5ab878c67d25f8 in 1324ms, sequenceid=87, compaction requested=true 2024-11-26T15:26:42,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:42,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef027b3b2a337d7b7f5ab878c67d25f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:26:42,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:42,918 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-26T15:26:42,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef027b3b2a337d7b7f5ab878c67d25f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:26:42,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:42,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef027b3b2a337d7b7f5ab878c67d25f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:26:42,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:42,918 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-26T15:26:42,919 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-26T15:26:42,919 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 189147 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-26T15:26:42,919 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ef027b3b2a337d7b7f5ab878c67d25f8/B is initiating minor compaction (all files) 2024-11-26T15:26:42,919 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): ef027b3b2a337d7b7f5ab878c67d25f8/A is initiating minor compaction (all files) 2024-11-26T15:26:42,919 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef027b3b2a337d7b7f5ab878c67d25f8/B in TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:42,919 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef027b3b2a337d7b7f5ab878c67d25f8/A in TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:42,919 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/5c0a2828eb334c8894434a262680d3a9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a27c1eefeddf4520aa0420183801682c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/bca6d2aaac5b44b7b2d52e331261b0fd, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/73fbf91aa56f47c997a5106e7c45ae04, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/9b52c9092a564b5ea10c92a7347f890f] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp, totalSize=184.7 K 2024-11-26T15:26:42,919 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/267854020ddc461bae9382b52ba088dd, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/96be9cabf86b446282c524b3e8b9b2e5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/80ab99748e6e47d4a4dc03c862f3d6d9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/1b3f45d976834053bb8808220b78cd2b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/d7a8222c250442ffb7e26300a2eca345] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp, totalSize=58.6 K 2024-11-26T15:26:42,919 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:42,920 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/5c0a2828eb334c8894434a262680d3a9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a27c1eefeddf4520aa0420183801682c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/bca6d2aaac5b44b7b2d52e331261b0fd, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/73fbf91aa56f47c997a5106e7c45ae04, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/9b52c9092a564b5ea10c92a7347f890f] 2024-11-26T15:26:42,920 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 267854020ddc461bae9382b52ba088dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732634796027 2024-11-26T15:26:42,920 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c0a2828eb334c8894434a262680d3a9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732634796027 2024-11-26T15:26:42,920 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 96be9cabf86b446282c524b3e8b9b2e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732634796071 2024-11-26T15:26:42,920 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting a27c1eefeddf4520aa0420183801682c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732634796071 2024-11-26T15:26:42,920 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 80ab99748e6e47d4a4dc03c862f3d6d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732634797305 2024-11-26T15:26:42,920 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting bca6d2aaac5b44b7b2d52e331261b0fd, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732634797244 2024-11-26T15:26:42,921 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b3f45d976834053bb8808220b78cd2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732634797382 2024-11-26T15:26:42,921 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73fbf91aa56f47c997a5106e7c45ae04, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732634797382 2024-11-26T15:26:42,921 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting d7a8222c250442ffb7e26300a2eca345, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732634799564 2024-11-26T15:26:42,921 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b52c9092a564b5ea10c92a7347f890f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732634799563 2024-11-26T15:26:42,928 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef027b3b2a337d7b7f5ab878c67d25f8#B#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:42,928 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/c9a61454da60497c91475d390228c864 is 50, key is test_row_0/B:col10/1732634801591/Put/seqid=0 2024-11-26T15:26:42,930 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:42,937 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126422e721bb0a54b8f81964e1e9b9b8073_ef027b3b2a337d7b7f5ab878c67d25f8 store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:42,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742188_1364 (size=12173) 2024-11-26T15:26:42,940 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126422e721bb0a54b8f81964e1e9b9b8073_ef027b3b2a337d7b7f5ab878c67d25f8, store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:42,940 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126422e721bb0a54b8f81964e1e9b9b8073_ef027b3b2a337d7b7f5ab878c67d25f8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:42,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742189_1365 (size=4469) 2024-11-26T15:26:42,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T15:26:42,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:42,999 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-26T15:26:43,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265f2ff73c6e6749ee882c94387a1781f5_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634801746/Put/seqid=0 2024-11-26T15:26:43,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742190_1366 (size=12154) 2024-11-26T15:26:43,344 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/c9a61454da60497c91475d390228c864 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/c9a61454da60497c91475d390228c864 2024-11-26T15:26:43,348 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef027b3b2a337d7b7f5ab878c67d25f8#A#compaction#309 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:43,349 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/4d08254c63ac42d4a999265ba381edec is 175, key is test_row_0/A:col10/1732634801591/Put/seqid=0 2024-11-26T15:26:43,353 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ef027b3b2a337d7b7f5ab878c67d25f8/B of ef027b3b2a337d7b7f5ab878c67d25f8 into c9a61454da60497c91475d390228c864(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:43,354 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:43,354 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., storeName=ef027b3b2a337d7b7f5ab878c67d25f8/B, priority=11, startTime=1732634802918; duration=0sec 2024-11-26T15:26:43,354 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:43,354 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef027b3b2a337d7b7f5ab878c67d25f8:B 2024-11-26T15:26:43,354 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-26T15:26:43,355 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-26T15:26:43,355 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ef027b3b2a337d7b7f5ab878c67d25f8/C is initiating minor compaction (all files) 2024-11-26T15:26:43,356 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef027b3b2a337d7b7f5ab878c67d25f8/C in TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:43,358 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/9e89209d90a04467a2cfa4ccf516bb29, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/821d7cdd62d2474183358ab803b785be, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/89bdfe3150a1400f83e0485025580a5a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/e8a56155e2ba4296b70267e5ceb42347, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3062de8224454b49846c0df66b0a27ca] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp, totalSize=58.6 K 2024-11-26T15:26:43,359 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e89209d90a04467a2cfa4ccf516bb29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732634796027 2024-11-26T15:26:43,359 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 821d7cdd62d2474183358ab803b785be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732634796071 2024-11-26T15:26:43,359 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 89bdfe3150a1400f83e0485025580a5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732634797305 2024-11-26T15:26:43,359 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting e8a56155e2ba4296b70267e5ceb42347, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732634797382 2024-11-26T15:26:43,360 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 3062de8224454b49846c0df66b0a27ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732634799564 2024-11-26T15:26:43,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742191_1367 (size=31127) 2024-11-26T15:26:43,376 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/4d08254c63ac42d4a999265ba381edec as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4d08254c63ac42d4a999265ba381edec 2024-11-26T15:26:43,383 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef027b3b2a337d7b7f5ab878c67d25f8#C#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:43,383 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/d7809ed92b504d43b1459ad807788569 is 50, key is test_row_0/C:col10/1732634801591/Put/seqid=0 2024-11-26T15:26:43,385 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ef027b3b2a337d7b7f5ab878c67d25f8/A of ef027b3b2a337d7b7f5ab878c67d25f8 into 4d08254c63ac42d4a999265ba381edec(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:43,385 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:43,385 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., storeName=ef027b3b2a337d7b7f5ab878c67d25f8/A, priority=11, startTime=1732634802917; duration=0sec 2024-11-26T15:26:43,386 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:43,386 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef027b3b2a337d7b7f5ab878c67d25f8:A 2024-11-26T15:26:43,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742192_1368 (size=12173) 2024-11-26T15:26:43,406 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/d7809ed92b504d43b1459ad807788569 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/d7809ed92b504d43b1459ad807788569 2024-11-26T15:26:43,412 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ef027b3b2a337d7b7f5ab878c67d25f8/C of ef027b3b2a337d7b7f5ab878c67d25f8 into d7809ed92b504d43b1459ad807788569(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:43,412 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:43,412 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., storeName=ef027b3b2a337d7b7f5ab878c67d25f8/C, priority=11, startTime=1732634802918; duration=0sec 2024-11-26T15:26:43,412 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:43,412 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef027b3b2a337d7b7f5ab878c67d25f8:C 2024-11-26T15:26:43,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:43,425 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265f2ff73c6e6749ee882c94387a1781f5_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f2ff73c6e6749ee882c94387a1781f5_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:43,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/de95275812ba4a1b99941c02e3528f5a, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:43,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/de95275812ba4a1b99941c02e3528f5a is 175, key is test_row_0/A:col10/1732634801746/Put/seqid=0 2024-11-26T15:26:43,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742193_1369 (size=30955) 2024-11-26T15:26:43,442 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/de95275812ba4a1b99941c02e3528f5a 2024-11-26T15:26:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/5350372298f14bac9bc38928693e617d is 50, key is test_row_0/B:col10/1732634801746/Put/seqid=0 2024-11-26T15:26:43,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742194_1370 (size=12001) 2024-11-26T15:26:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T15:26:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:43,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:43,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634863811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:43,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:43,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:43,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:43,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634863812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:43,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:43,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:43,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:43,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634863817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:43,888 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/5350372298f14bac9bc38928693e617d 2024-11-26T15:26:43,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/2e3b3fa4d44940dfabf29a3a16a28fa5 is 50, key is test_row_0/C:col10/1732634801746/Put/seqid=0 2024-11-26T15:26:43,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742195_1371 (size=12001) 2024-11-26T15:26:43,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634863922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:43,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634863922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:43,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634863922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:43,925 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in ef027b3b2a337d7b7f5ab878c67d25f8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:43,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634863932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:43,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634863932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:43,935 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in ef027b3b2a337d7b7f5ab878c67d25f8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:44,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634864132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:44,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634864132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:44,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634864142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:44,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:44,301 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/2e3b3fa4d44940dfabf29a3a16a28fa5 2024-11-26T15:26:44,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/de95275812ba4a1b99941c02e3528f5a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/de95275812ba4a1b99941c02e3528f5a 2024-11-26T15:26:44,307 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/de95275812ba4a1b99941c02e3528f5a, entries=150, sequenceid=112, filesize=30.2 K 2024-11-26T15:26:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/5350372298f14bac9bc38928693e617d as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/5350372298f14bac9bc38928693e617d 2024-11-26T15:26:44,311 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/5350372298f14bac9bc38928693e617d, entries=150, sequenceid=112, filesize=11.7 K 2024-11-26T15:26:44,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/2e3b3fa4d44940dfabf29a3a16a28fa5 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/2e3b3fa4d44940dfabf29a3a16a28fa5 2024-11-26T15:26:44,315 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/2e3b3fa4d44940dfabf29a3a16a28fa5, entries=150, sequenceid=112, filesize=11.7 K 2024-11-26T15:26:44,316 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ef027b3b2a337d7b7f5ab878c67d25f8 in 1317ms, sequenceid=112, compaction requested=false 2024-11-26T15:26:44,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:44,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:44,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-26T15:26:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-26T15:26:44,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-26T15:26:44,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8510 sec 2024-11-26T15:26:44,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.8560 sec 2024-11-26T15:26:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:44,446 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:44,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-26T15:26:44,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126010b4ea3cde24f0ea1a3a9843f20eca2_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634804446/Put/seqid=0 2024-11-26T15:26:44,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742196_1372 (size=14594) 2024-11-26T15:26:44,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634864497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:44,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:44,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634864503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:44,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:44,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:44,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634864504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:44,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:44,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:44,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634864613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634864613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:44,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:44,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:44,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634864622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:44,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:44,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634864833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:44,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:44,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634864833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634864833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:44,858 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:44,861 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126010b4ea3cde24f0ea1a3a9843f20eca2_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126010b4ea3cde24f0ea1a3a9843f20eca2_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:44,862 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/3dcd0eceab5c4663b75270da14254578, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:44,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/3dcd0eceab5c4663b75270da14254578 is 175, key is test_row_0/A:col10/1732634804446/Put/seqid=0 2024-11-26T15:26:44,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742197_1373 (size=39549) 2024-11-26T15:26:45,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634865142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:45,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634865142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:45,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:45,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634865152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:45,266 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/3dcd0eceab5c4663b75270da14254578 2024-11-26T15:26:45,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/ad65bc43bd0c4ada90b18eedbed92520 is 50, key is test_row_0/B:col10/1732634804446/Put/seqid=0 2024-11-26T15:26:45,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742198_1374 (size=12001) 2024-11-26T15:26:45,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T15:26:45,633 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:45,633 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-26T15:26:45,636 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T15:26:45,637 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:45,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:45,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634865653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:45,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:45,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:45,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:45,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634865653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:45,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634865672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:45,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/ad65bc43bd0c4ada90b18eedbed92520 2024-11-26T15:26:45,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/5f8cdd18e56948b290a1a591c9460073 is 50, key is test_row_0/C:col10/1732634804446/Put/seqid=0 2024-11-26T15:26:45,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742199_1375 (size=12001) 2024-11-26T15:26:45,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/5f8cdd18e56948b290a1a591c9460073 2024-11-26T15:26:45,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/3dcd0eceab5c4663b75270da14254578 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/3dcd0eceab5c4663b75270da14254578 2024-11-26T15:26:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T15:26:45,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/3dcd0eceab5c4663b75270da14254578, entries=200, sequenceid=128, filesize=38.6 K 2024-11-26T15:26:45,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/ad65bc43bd0c4ada90b18eedbed92520 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/ad65bc43bd0c4ada90b18eedbed92520 2024-11-26T15:26:45,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/ad65bc43bd0c4ada90b18eedbed92520, entries=150, sequenceid=128, filesize=11.7 K 2024-11-26T15:26:45,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/5f8cdd18e56948b290a1a591c9460073 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/5f8cdd18e56948b290a1a591c9460073 2024-11-26T15:26:45,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/5f8cdd18e56948b290a1a591c9460073, entries=150, sequenceid=128, filesize=11.7 K 2024-11-26T15:26:45,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ef027b3b2a337d7b7f5ab878c67d25f8 in 1317ms, sequenceid=128, compaction requested=true 2024-11-26T15:26:45,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:45,763 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:45,765 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101631 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:45,765 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): ef027b3b2a337d7b7f5ab878c67d25f8/A is initiating minor compaction (all files) 2024-11-26T15:26:45,765 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef027b3b2a337d7b7f5ab878c67d25f8/A in TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:45,765 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4d08254c63ac42d4a999265ba381edec, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/de95275812ba4a1b99941c02e3528f5a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/3dcd0eceab5c4663b75270da14254578] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp, totalSize=99.2 K 2024-11-26T15:26:45,765 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:45,765 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4d08254c63ac42d4a999265ba381edec, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/de95275812ba4a1b99941c02e3528f5a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/3dcd0eceab5c4663b75270da14254578] 2024-11-26T15:26:45,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef027b3b2a337d7b7f5ab878c67d25f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:26:45,766 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d08254c63ac42d4a999265ba381edec, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732634799564 2024-11-26T15:26:45,766 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting de95275812ba4a1b99941c02e3528f5a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732634801714 2024-11-26T15:26:45,766 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dcd0eceab5c4663b75270da14254578, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732634803805 2024-11-26T15:26:45,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:45,767 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:45,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef027b3b2a337d7b7f5ab878c67d25f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:26:45,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:45,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef027b3b2a337d7b7f5ab878c67d25f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:26:45,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:45,768 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:45,768 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ef027b3b2a337d7b7f5ab878c67d25f8/B is initiating minor compaction (all files) 2024-11-26T15:26:45,769 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef027b3b2a337d7b7f5ab878c67d25f8/B in TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:45,769 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/c9a61454da60497c91475d390228c864, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/5350372298f14bac9bc38928693e617d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/ad65bc43bd0c4ada90b18eedbed92520] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp, totalSize=35.3 K 2024-11-26T15:26:45,773 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting c9a61454da60497c91475d390228c864, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732634799564 2024-11-26T15:26:45,776 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 5350372298f14bac9bc38928693e617d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732634801714 2024-11-26T15:26:45,777 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting ad65bc43bd0c4ada90b18eedbed92520, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732634803810 2024-11-26T15:26:45,790 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:45,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-26T15:26:45,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:45,791 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-26T15:26:45,802 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef027b3b2a337d7b7f5ab878c67d25f8#B#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:45,803 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/dc51efb5e075480980c7e9563676b1bf is 50, key is test_row_0/B:col10/1732634804446/Put/seqid=0 2024-11-26T15:26:45,811 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411268efd17391e6649cbbe8adc283956e499_ef027b3b2a337d7b7f5ab878c67d25f8 store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:45,813 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411268efd17391e6649cbbe8adc283956e499_ef027b3b2a337d7b7f5ab878c67d25f8, store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:45,813 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268efd17391e6649cbbe8adc283956e499_ef027b3b2a337d7b7f5ab878c67d25f8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:45,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267046b3d397ef4c7eace6813a6c906c16_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634804495/Put/seqid=0 2024-11-26T15:26:45,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742200_1376 (size=12275) 2024-11-26T15:26:45,878 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/dc51efb5e075480980c7e9563676b1bf as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/dc51efb5e075480980c7e9563676b1bf 2024-11-26T15:26:45,884 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef027b3b2a337d7b7f5ab878c67d25f8/B of ef027b3b2a337d7b7f5ab878c67d25f8 into dc51efb5e075480980c7e9563676b1bf(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:45,884 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:45,884 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., storeName=ef027b3b2a337d7b7f5ab878c67d25f8/B, priority=13, startTime=1732634805767; duration=0sec 2024-11-26T15:26:45,885 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:45,885 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef027b3b2a337d7b7f5ab878c67d25f8:B 2024-11-26T15:26:45,885 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:45,885 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:45,886 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ef027b3b2a337d7b7f5ab878c67d25f8/C is initiating minor compaction (all files) 2024-11-26T15:26:45,886 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef027b3b2a337d7b7f5ab878c67d25f8/C in TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:45,886 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/d7809ed92b504d43b1459ad807788569, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/2e3b3fa4d44940dfabf29a3a16a28fa5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/5f8cdd18e56948b290a1a591c9460073] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp, totalSize=35.3 K 2024-11-26T15:26:45,886 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting d7809ed92b504d43b1459ad807788569, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732634799564 2024-11-26T15:26:45,886 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e3b3fa4d44940dfabf29a3a16a28fa5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732634801714 2024-11-26T15:26:45,886 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f8cdd18e56948b290a1a591c9460073, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732634803810 2024-11-26T15:26:45,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742201_1377 (size=4469) 2024-11-26T15:26:45,896 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef027b3b2a337d7b7f5ab878c67d25f8#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:45,897 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/925af371837848649b3daa192e69cfa8 is 50, key is test_row_0/C:col10/1732634804446/Put/seqid=0 2024-11-26T15:26:45,898 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef027b3b2a337d7b7f5ab878c67d25f8#A#compaction#317 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:45,899 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/d3487054f2ef479ab31dc4e3c45655b1 is 175, key is test_row_0/A:col10/1732634804446/Put/seqid=0 2024-11-26T15:26:45,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742202_1378 (size=12304) 2024-11-26T15:26:45,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742203_1379 (size=31229) 2024-11-26T15:26:45,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T15:26:45,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742204_1380 (size=12275) 2024-11-26T15:26:45,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:45,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634865987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634865990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:46,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634866103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:46,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:46,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634866103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:46,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T15:26:46,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634866314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:46,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634866314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:46,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:46,329 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267046b3d397ef4c7eace6813a6c906c16_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267046b3d397ef4c7eace6813a6c906c16_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:46,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/27033cf6211f47ae8bc3c31c82f3c05e, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:46,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/27033cf6211f47ae8bc3c31c82f3c05e is 175, key is test_row_0/A:col10/1732634804495/Put/seqid=0 2024-11-26T15:26:46,342 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/d3487054f2ef479ab31dc4e3c45655b1 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/d3487054f2ef479ab31dc4e3c45655b1 2024-11-26T15:26:46,347 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef027b3b2a337d7b7f5ab878c67d25f8/A of ef027b3b2a337d7b7f5ab878c67d25f8 into d3487054f2ef479ab31dc4e3c45655b1(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:46,347 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:46,347 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., storeName=ef027b3b2a337d7b7f5ab878c67d25f8/A, priority=13, startTime=1732634805763; duration=0sec 2024-11-26T15:26:46,348 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:46,348 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef027b3b2a337d7b7f5ab878c67d25f8:A 2024-11-26T15:26:46,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742205_1381 (size=31105) 2024-11-26T15:26:46,363 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/925af371837848649b3daa192e69cfa8 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/925af371837848649b3daa192e69cfa8 2024-11-26T15:26:46,372 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef027b3b2a337d7b7f5ab878c67d25f8/C of ef027b3b2a337d7b7f5ab878c67d25f8 into 925af371837848649b3daa192e69cfa8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:46,372 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:46,372 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., storeName=ef027b3b2a337d7b7f5ab878c67d25f8/C, priority=13, startTime=1732634805768; duration=0sec 2024-11-26T15:26:46,372 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:46,372 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef027b3b2a337d7b7f5ab878c67d25f8:C 2024-11-26T15:26:46,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634866633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:46,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634866633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:46,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634866662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:46,670 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in ef027b3b2a337d7b7f5ab878c67d25f8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:46,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:46,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634866674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:46,680 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in ef027b3b2a337d7b7f5ab878c67d25f8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:46,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634866692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:46,702 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in ef027b3b2a337d7b7f5ab878c67d25f8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:46,757 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/27033cf6211f47ae8bc3c31c82f3c05e 2024-11-26T15:26:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T15:26:46,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/7c73a1508cab412c964d193468350cad is 50, key is test_row_0/B:col10/1732634804495/Put/seqid=0 2024-11-26T15:26:46,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742206_1382 (size=12151) 2024-11-26T15:26:47,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:47,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634867152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:47,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:47,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634867153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:47,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:47,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:47,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:47,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:47,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:47,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:47,235 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/7c73a1508cab412c964d193468350cad 2024-11-26T15:26:47,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/3bb63b491b914e11a0910d3b01771d6e is 50, key is test_row_0/C:col10/1732634804495/Put/seqid=0 2024-11-26T15:26:47,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742207_1383 (size=12151) 2024-11-26T15:26:47,296 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/3bb63b491b914e11a0910d3b01771d6e 2024-11-26T15:26:47,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/27033cf6211f47ae8bc3c31c82f3c05e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/27033cf6211f47ae8bc3c31c82f3c05e 2024-11-26T15:26:47,309 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/27033cf6211f47ae8bc3c31c82f3c05e, entries=150, sequenceid=151, filesize=30.4 K 2024-11-26T15:26:47,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/7c73a1508cab412c964d193468350cad as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/7c73a1508cab412c964d193468350cad 2024-11-26T15:26:47,316 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/7c73a1508cab412c964d193468350cad, entries=150, sequenceid=151, filesize=11.9 K 2024-11-26T15:26:47,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/3bb63b491b914e11a0910d3b01771d6e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3bb63b491b914e11a0910d3b01771d6e 2024-11-26T15:26:47,321 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3bb63b491b914e11a0910d3b01771d6e, entries=150, sequenceid=151, filesize=11.9 K 2024-11-26T15:26:47,322 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for ef027b3b2a337d7b7f5ab878c67d25f8 in 1531ms, sequenceid=151, compaction requested=false 2024-11-26T15:26:47,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:47,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:47,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-26T15:26:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-26T15:26:47,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-26T15:26:47,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6890 sec 2024-11-26T15:26:47,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.6980 sec 2024-11-26T15:26:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T15:26:47,783 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:47,783 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-26T15:26:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T15:26:47,785 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:47,786 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:47,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T15:26:47,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-26T15:26:47,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:47,944 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-26T15:26:47,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263c79c008bbba41bf9070c46af1e78129_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634805989/Put/seqid=0 2024-11-26T15:26:47,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742208_1384 (size=12304) 2024-11-26T15:26:47,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:47,980 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263c79c008bbba41bf9070c46af1e78129_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263c79c008bbba41bf9070c46af1e78129_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:47,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/7586f9633a6a4d0a8fc02de12436eada, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:47,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/7586f9633a6a4d0a8fc02de12436eada is 175, key is test_row_0/A:col10/1732634805989/Put/seqid=0 2024-11-26T15:26:48,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742209_1385 (size=31105) 2024-11-26T15:26:48,025 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/7586f9633a6a4d0a8fc02de12436eada 2024-11-26T15:26:48,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/45ecdfc2816947d79bb8d68302534b8a is 50, key is test_row_0/B:col10/1732634805989/Put/seqid=0 2024-11-26T15:26:48,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742210_1386 (size=12151) 2024-11-26T15:26:48,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T15:26:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:48,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:48,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634868241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:48,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634868242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:48,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:48,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634868353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634868353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T15:26:48,453 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/45ecdfc2816947d79bb8d68302534b8a 2024-11-26T15:26:48,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/9caf73cdc9684c95b62faf52b17f6e36 is 50, key is test_row_0/C:col10/1732634805989/Put/seqid=0 2024-11-26T15:26:48,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742211_1387 (size=12151) 2024-11-26T15:26:48,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634868563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:48,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634868563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:48,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634868683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:48,693 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in ef027b3b2a337d7b7f5ab878c67d25f8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:48,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:48,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634868702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:48,708 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in ef027b3b2a337d7b7f5ab878c67d25f8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:48,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634868712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:48,719 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in ef027b3b2a337d7b7f5ab878c67d25f8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:48,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634868873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:48,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634868873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:48,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:48,891 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/9caf73cdc9684c95b62faf52b17f6e36 2024-11-26T15:26:48,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/7586f9633a6a4d0a8fc02de12436eada as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/7586f9633a6a4d0a8fc02de12436eada 2024-11-26T15:26:48,905 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/7586f9633a6a4d0a8fc02de12436eada, entries=150, sequenceid=167, filesize=30.4 K 2024-11-26T15:26:48,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/45ecdfc2816947d79bb8d68302534b8a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/45ecdfc2816947d79bb8d68302534b8a 2024-11-26T15:26:48,911 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/45ecdfc2816947d79bb8d68302534b8a, entries=150, sequenceid=167, filesize=11.9 K 2024-11-26T15:26:48,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/9caf73cdc9684c95b62faf52b17f6e36 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/9caf73cdc9684c95b62faf52b17f6e36 2024-11-26T15:26:48,917 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/9caf73cdc9684c95b62faf52b17f6e36, entries=150, sequenceid=167, filesize=11.9 K 2024-11-26T15:26:48,917 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ef027b3b2a337d7b7f5ab878c67d25f8 in 973ms, sequenceid=167, compaction requested=true 2024-11-26T15:26:48,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:48,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:48,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-26T15:26:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-26T15:26:48,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-26T15:26:48,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1330 sec 2024-11-26T15:26:48,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.1380 sec 2024-11-26T15:26:48,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-26T15:26:48,923 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:48,923 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-26T15:26:48,927 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T15:26:48,930 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:48,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T15:26:49,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-26T15:26:49,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:49,083 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-26T15:26:49,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126510a670f38de42d0a78e940ef73f0337_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634808241/Put/seqid=0 2024-11-26T15:26:49,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742212_1388 (size=12304) 2024-11-26T15:26:49,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:49,127 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126510a670f38de42d0a78e940ef73f0337_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126510a670f38de42d0a78e940ef73f0337_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:49,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/a02d0354ae29469c87baa6df95396665, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:49,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/a02d0354ae29469c87baa6df95396665 is 175, key is test_row_0/A:col10/1732634808241/Put/seqid=0 2024-11-26T15:26:49,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742213_1389 (size=31105) 2024-11-26T15:26:49,149 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/a02d0354ae29469c87baa6df95396665 2024-11-26T15:26:49,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/43ecbaaf27594b3daa8a87c2d554a070 is 50, key is test_row_0/B:col10/1732634808241/Put/seqid=0 2024-11-26T15:26:49,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742214_1390 (size=12151) 2024-11-26T15:26:49,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T15:26:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:49,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:49,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634869442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:49,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634869443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:49,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:49,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:49,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T15:26:49,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634869552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:49,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:49,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:49,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:49,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634869562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:49,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:49,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:49,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:49,579 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/43ecbaaf27594b3daa8a87c2d554a070 2024-11-26T15:26:49,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/1c13f7efd87f484687dba7278c750edb is 50, key is test_row_0/C:col10/1732634808241/Put/seqid=0 2024-11-26T15:26:49,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742215_1391 (size=12151) 2024-11-26T15:26:49,590 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/1c13f7efd87f484687dba7278c750edb 2024-11-26T15:26:49,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/a02d0354ae29469c87baa6df95396665 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a02d0354ae29469c87baa6df95396665 2024-11-26T15:26:49,597 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a02d0354ae29469c87baa6df95396665, entries=150, sequenceid=190, filesize=30.4 K 2024-11-26T15:26:49,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/43ecbaaf27594b3daa8a87c2d554a070 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/43ecbaaf27594b3daa8a87c2d554a070 2024-11-26T15:26:49,600 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/43ecbaaf27594b3daa8a87c2d554a070, entries=150, sequenceid=190, filesize=11.9 K 2024-11-26T15:26:49,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/1c13f7efd87f484687dba7278c750edb as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/1c13f7efd87f484687dba7278c750edb 2024-11-26T15:26:49,604 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/1c13f7efd87f484687dba7278c750edb, entries=150, sequenceid=190, filesize=11.9 K 2024-11-26T15:26:49,604 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for ef027b3b2a337d7b7f5ab878c67d25f8 in 521ms, sequenceid=190, compaction requested=true 2024-11-26T15:26:49,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:49,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:49,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-26T15:26:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-26T15:26:49,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-26T15:26:49,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 676 msec 2024-11-26T15:26:49,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 684 msec 2024-11-26T15:26:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:49,766 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:49,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-26T15:26:49,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269fc9bd1f571743489a61390f8f930401_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634809434/Put/seqid=0 2024-11-26T15:26:49,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742216_1392 (size=14794) 2024-11-26T15:26:49,823 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:49,832 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269fc9bd1f571743489a61390f8f930401_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269fc9bd1f571743489a61390f8f930401_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:49,833 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/655cafc6326b4ff4844b1e1d2ee6f26e, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:49,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/655cafc6326b4ff4844b1e1d2ee6f26e is 175, key is test_row_0/A:col10/1732634809434/Put/seqid=0 2024-11-26T15:26:49,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742217_1393 (size=39749) 2024-11-26T15:26:49,845 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/655cafc6326b4ff4844b1e1d2ee6f26e 2024-11-26T15:26:49,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/e6f5802ac82441c799ab4afc96d265d9 is 50, key is test_row_0/B:col10/1732634809434/Put/seqid=0 2024-11-26T15:26:49,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742218_1394 (size=12151) 2024-11-26T15:26:49,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/e6f5802ac82441c799ab4afc96d265d9 2024-11-26T15:26:49,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/65477bc3499f4a08bf2837556cea32a7 is 50, key is test_row_0/C:col10/1732634809434/Put/seqid=0 2024-11-26T15:26:49,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742219_1395 (size=12151) 2024-11-26T15:26:49,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/65477bc3499f4a08bf2837556cea32a7 2024-11-26T15:26:49,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/655cafc6326b4ff4844b1e1d2ee6f26e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/655cafc6326b4ff4844b1e1d2ee6f26e 2024-11-26T15:26:49,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634869901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:49,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:49,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:49,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:49,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/655cafc6326b4ff4844b1e1d2ee6f26e, entries=200, sequenceid=204, filesize=38.8 K 2024-11-26T15:26:49,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/e6f5802ac82441c799ab4afc96d265d9 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/e6f5802ac82441c799ab4afc96d265d9 2024-11-26T15:26:49,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/e6f5802ac82441c799ab4afc96d265d9, entries=150, sequenceid=204, filesize=11.9 K 2024-11-26T15:26:49,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/65477bc3499f4a08bf2837556cea32a7 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/65477bc3499f4a08bf2837556cea32a7 2024-11-26T15:26:49,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634869903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:49,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/65477bc3499f4a08bf2837556cea32a7, entries=150, sequenceid=204, filesize=11.9 K 2024-11-26T15:26:49,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for ef027b3b2a337d7b7f5ab878c67d25f8 in 157ms, sequenceid=204, compaction requested=true 2024-11-26T15:26:49,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:49,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef027b3b2a337d7b7f5ab878c67d25f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:26:49,924 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-26T15:26:49,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:49,924 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-26T15:26:49,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef027b3b2a337d7b7f5ab878c67d25f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:26:49,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:49,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef027b3b2a337d7b7f5ab878c67d25f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:26:49,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:49,925 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60879 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-26T15:26:49,925 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ef027b3b2a337d7b7f5ab878c67d25f8/B is initiating minor compaction (all files) 2024-11-26T15:26:49,925 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef027b3b2a337d7b7f5ab878c67d25f8/B in TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:49,925 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/dc51efb5e075480980c7e9563676b1bf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/7c73a1508cab412c964d193468350cad, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/45ecdfc2816947d79bb8d68302534b8a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/43ecbaaf27594b3daa8a87c2d554a070, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/e6f5802ac82441c799ab4afc96d265d9] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp, totalSize=59.5 K 2024-11-26T15:26:49,926 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 164293 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-26T15:26:49,926 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): ef027b3b2a337d7b7f5ab878c67d25f8/A is initiating minor compaction (all files) 2024-11-26T15:26:49,926 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting dc51efb5e075480980c7e9563676b1bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732634803810 2024-11-26T15:26:49,926 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef027b3b2a337d7b7f5ab878c67d25f8/A in TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:49,926 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/d3487054f2ef479ab31dc4e3c45655b1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/27033cf6211f47ae8bc3c31c82f3c05e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/7586f9633a6a4d0a8fc02de12436eada, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a02d0354ae29469c87baa6df95396665, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/655cafc6326b4ff4844b1e1d2ee6f26e] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp, totalSize=160.4 K 2024-11-26T15:26:49,926 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:49,926 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/d3487054f2ef479ab31dc4e3c45655b1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/27033cf6211f47ae8bc3c31c82f3c05e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/7586f9633a6a4d0a8fc02de12436eada, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a02d0354ae29469c87baa6df95396665, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/655cafc6326b4ff4844b1e1d2ee6f26e] 2024-11-26T15:26:49,926 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c73a1508cab412c964d193468350cad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732634804495 2024-11-26T15:26:49,926 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3487054f2ef479ab31dc4e3c45655b1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732634803810 2024-11-26T15:26:49,926 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 45ecdfc2816947d79bb8d68302534b8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732634805964 2024-11-26T15:26:49,926 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27033cf6211f47ae8bc3c31c82f3c05e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732634804495 2024-11-26T15:26:49,927 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 43ecbaaf27594b3daa8a87c2d554a070, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732634808205 2024-11-26T15:26:49,927 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7586f9633a6a4d0a8fc02de12436eada, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732634805964 2024-11-26T15:26:49,927 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting e6f5802ac82441c799ab4afc96d265d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732634809434 2024-11-26T15:26:49,927 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting a02d0354ae29469c87baa6df95396665, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732634808205 2024-11-26T15:26:49,927 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 655cafc6326b4ff4844b1e1d2ee6f26e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732634809430 2024-11-26T15:26:49,942 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:49,944 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef027b3b2a337d7b7f5ab878c67d25f8#B#compaction#332 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:49,944 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/8057edaa647b4a0db4b30906d395a4fc is 50, key is test_row_0/B:col10/1732634809434/Put/seqid=0 2024-11-26T15:26:49,945 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411264a06fa9aabff440685647db6df5d3422_ef027b3b2a337d7b7f5ab878c67d25f8 store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:49,948 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411264a06fa9aabff440685647db6df5d3422_ef027b3b2a337d7b7f5ab878c67d25f8, store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:49,948 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264a06fa9aabff440685647db6df5d3422_ef027b3b2a337d7b7f5ab878c67d25f8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:49,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742221_1397 (size=4469) 2024-11-26T15:26:49,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742220_1396 (size=12595) 2024-11-26T15:26:49,967 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/8057edaa647b4a0db4b30906d395a4fc as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/8057edaa647b4a0db4b30906d395a4fc 2024-11-26T15:26:49,972 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ef027b3b2a337d7b7f5ab878c67d25f8/B of ef027b3b2a337d7b7f5ab878c67d25f8 into 8057edaa647b4a0db4b30906d395a4fc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:49,972 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:49,972 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., storeName=ef027b3b2a337d7b7f5ab878c67d25f8/B, priority=11, startTime=1732634809924; duration=0sec 2024-11-26T15:26:49,972 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:49,972 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef027b3b2a337d7b7f5ab878c67d25f8:B 2024-11-26T15:26:49,973 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-26T15:26:49,974 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60879 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-26T15:26:49,974 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ef027b3b2a337d7b7f5ab878c67d25f8/C is initiating minor compaction (all files) 2024-11-26T15:26:49,974 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef027b3b2a337d7b7f5ab878c67d25f8/C in TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:49,974 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/925af371837848649b3daa192e69cfa8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3bb63b491b914e11a0910d3b01771d6e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/9caf73cdc9684c95b62faf52b17f6e36, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/1c13f7efd87f484687dba7278c750edb, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/65477bc3499f4a08bf2837556cea32a7] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp, totalSize=59.5 K 2024-11-26T15:26:49,975 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 925af371837848649b3daa192e69cfa8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732634803810 2024-11-26T15:26:49,975 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bb63b491b914e11a0910d3b01771d6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732634804495 2024-11-26T15:26:49,975 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 9caf73cdc9684c95b62faf52b17f6e36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732634805964 2024-11-26T15:26:49,976 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c13f7efd87f484687dba7278c750edb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732634808205 2024-11-26T15:26:49,976 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 65477bc3499f4a08bf2837556cea32a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732634809434 2024-11-26T15:26:49,984 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef027b3b2a337d7b7f5ab878c67d25f8#C#compaction#334 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:49,984 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/487e081861104f479bf40c10ec8c1382 is 50, key is test_row_0/C:col10/1732634809434/Put/seqid=0 2024-11-26T15:26:49,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742222_1398 (size=12595) 2024-11-26T15:26:49,995 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/487e081861104f479bf40c10ec8c1382 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/487e081861104f479bf40c10ec8c1382 2024-11-26T15:26:50,006 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ef027b3b2a337d7b7f5ab878c67d25f8/C of ef027b3b2a337d7b7f5ab878c67d25f8 into 487e081861104f479bf40c10ec8c1382(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:50,006 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:50,006 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., storeName=ef027b3b2a337d7b7f5ab878c67d25f8/C, priority=11, startTime=1732634809924; duration=0sec 2024-11-26T15:26:50,006 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:50,006 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef027b3b2a337d7b7f5ab878c67d25f8:C 2024-11-26T15:26:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:50,019 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:50,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-26T15:26:50,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b50bd8b3d44b4517a6d2c9d5a6807ce8_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634809901/Put/seqid=0 2024-11-26T15:26:50,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742223_1399 (size=14794) 2024-11-26T15:26:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T15:26:50,062 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:50,063 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-26T15:26:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T15:26:50,064 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:50,064 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:50,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:50,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634870054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:50,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:50,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:50,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:50,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634870062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T15:26:50,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634870172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:50,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:50,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:50,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634870182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:50,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-26T15:26:50,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:50,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:50,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:50,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,362 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef027b3b2a337d7b7f5ab878c67d25f8#A#compaction#333 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:50,363 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/3d511b2afd024856b1eebcaefbc48efa is 175, key is test_row_0/A:col10/1732634809434/Put/seqid=0 2024-11-26T15:26:50,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-26T15:26:50,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:50,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:50,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:50,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T15:26:50,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742224_1400 (size=31549) 2024-11-26T15:26:50,394 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/3d511b2afd024856b1eebcaefbc48efa as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/3d511b2afd024856b1eebcaefbc48efa 2024-11-26T15:26:50,400 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in ef027b3b2a337d7b7f5ab878c67d25f8/A of ef027b3b2a337d7b7f5ab878c67d25f8 into 3d511b2afd024856b1eebcaefbc48efa(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:50,400 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:50,401 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., storeName=ef027b3b2a337d7b7f5ab878c67d25f8/A, priority=11, startTime=1732634809923; duration=0sec 2024-11-26T15:26:50,401 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:50,401 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef027b3b2a337d7b7f5ab878c67d25f8:A 2024-11-26T15:26:50,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:50,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634870392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:50,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:50,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:50,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:50,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:50,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634870403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:50,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:50,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:50,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:50,434 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:50,442 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b50bd8b3d44b4517a6d2c9d5a6807ce8_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b50bd8b3d44b4517a6d2c9d5a6807ce8_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:50,447 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/1071f1bcb87549b5be26d7ca417b4731, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:50,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/1071f1bcb87549b5be26d7ca417b4731 is 175, key is test_row_0/A:col10/1732634809901/Put/seqid=0 2024-11-26T15:26:50,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742225_1401 (size=39749) 2024-11-26T15:26:50,465 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/1071f1bcb87549b5be26d7ca417b4731 2024-11-26T15:26:50,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/ca84c4954abf40a59feb94bb30f893bc is 50, key is test_row_0/B:col10/1732634809901/Put/seqid=0 2024-11-26T15:26:50,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742226_1402 (size=12151) 2024-11-26T15:26:50,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-26T15:26:50,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:50,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:50,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:50,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-26T15:26:50,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:50,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:50,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:50,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T15:26:50,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634870712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:50,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634870722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:50,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-26T15:26:50,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:50,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:50,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:50,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/ca84c4954abf40a59feb94bb30f893bc 2024-11-26T15:26:50,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/ef4b2e1396864a24b500da7938b7af4f is 50, key is test_row_0/C:col10/1732634809901/Put/seqid=0 2024-11-26T15:26:50,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742227_1403 (size=12151) 2024-11-26T15:26:50,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-26T15:26:50,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:50,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:50,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:50,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:51,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-26T15:26:51,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:51,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:51,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:51,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:51,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T15:26:51,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634871233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:51,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634871242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:51,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:51,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:51,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:51,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-26T15:26:51,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:51,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:51,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:51,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:51,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:26:51,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/ef4b2e1396864a24b500da7938b7af4f 2024-11-26T15:26:51,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/1071f1bcb87549b5be26d7ca417b4731 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/1071f1bcb87549b5be26d7ca417b4731 2024-11-26T15:26:51,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/1071f1bcb87549b5be26d7ca417b4731, entries=200, sequenceid=229, filesize=38.8 K 2024-11-26T15:26:51,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/ca84c4954abf40a59feb94bb30f893bc as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/ca84c4954abf40a59feb94bb30f893bc 2024-11-26T15:26:51,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/ca84c4954abf40a59feb94bb30f893bc, entries=150, sequenceid=229, filesize=11.9 K 2024-11-26T15:26:51,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/ef4b2e1396864a24b500da7938b7af4f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/ef4b2e1396864a24b500da7938b7af4f 2024-11-26T15:26:51,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/ef4b2e1396864a24b500da7938b7af4f, entries=150, sequenceid=229, filesize=11.9 K 2024-11-26T15:26:51,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for ef027b3b2a337d7b7f5ab878c67d25f8 in 1325ms, sequenceid=229, compaction requested=false 2024-11-26T15:26:51,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:51,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-26T15:26:51,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:51,448 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-26T15:26:51,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d6df1dc50c5541499cba42a3ad5aa1b7_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634810061/Put/seqid=0 2024-11-26T15:26:51,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742228_1404 (size=12304) 2024-11-26T15:26:51,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:51,494 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d6df1dc50c5541499cba42a3ad5aa1b7_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d6df1dc50c5541499cba42a3ad5aa1b7_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:51,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/09f31c336cfc4656886987d46b745e95, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:51,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/09f31c336cfc4656886987d46b745e95 is 175, key is test_row_0/A:col10/1732634810061/Put/seqid=0 2024-11-26T15:26:51,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742229_1405 (size=31105) 2024-11-26T15:26:51,900 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/09f31c336cfc4656886987d46b745e95 2024-11-26T15:26:51,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/2aa073bf6b87447e90ad38634766d667 is 50, key is test_row_0/B:col10/1732634810061/Put/seqid=0 2024-11-26T15:26:51,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742230_1406 (size=12151) 2024-11-26T15:26:51,939 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/2aa073bf6b87447e90ad38634766d667 2024-11-26T15:26:51,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/586028946bd84acb9e893ec91a9ea6f8 is 50, key is test_row_0/C:col10/1732634810061/Put/seqid=0 2024-11-26T15:26:51,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742231_1407 (size=12151) 2024-11-26T15:26:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T15:26:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:52,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:52,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634872312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:52,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:52,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634872315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:52,363 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/586028946bd84acb9e893ec91a9ea6f8 2024-11-26T15:26:52,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/09f31c336cfc4656886987d46b745e95 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/09f31c336cfc4656886987d46b745e95 2024-11-26T15:26:52,371 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/09f31c336cfc4656886987d46b745e95, entries=150, sequenceid=243, filesize=30.4 K 2024-11-26T15:26:52,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/2aa073bf6b87447e90ad38634766d667 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/2aa073bf6b87447e90ad38634766d667 2024-11-26T15:26:52,376 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/2aa073bf6b87447e90ad38634766d667, entries=150, sequenceid=243, filesize=11.9 K 2024-11-26T15:26:52,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/586028946bd84acb9e893ec91a9ea6f8 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/586028946bd84acb9e893ec91a9ea6f8 2024-11-26T15:26:52,380 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/586028946bd84acb9e893ec91a9ea6f8, entries=150, sequenceid=243, filesize=11.9 K 2024-11-26T15:26:52,380 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ef027b3b2a337d7b7f5ab878c67d25f8 in 932ms, sequenceid=243, compaction requested=true 2024-11-26T15:26:52,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:52,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:52,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-26T15:26:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-26T15:26:52,383 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-26T15:26:52,383 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3180 sec 2024-11-26T15:26:52,384 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.3200 sec 2024-11-26T15:26:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:52,428 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:52,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T15:26:52,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263aa18b61fad54797a54f2e7166430ad1_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634812427/Put/seqid=0 2024-11-26T15:26:52,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742232_1408 (size=14994) 2024-11-26T15:26:52,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634872470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:52,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634872471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:52,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634872583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:52,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634872583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:52,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53836 deadline: 1732634872722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:52,730 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in ef027b3b2a337d7b7f5ab878c67d25f8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:52,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:52,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1732634872752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:52,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53820 deadline: 1732634872752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:52,759 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in ef027b3b2a337d7b7f5ab878c67d25f8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:52,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,759 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in ef027b3b2a337d7b7f5ab878c67d25f8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:26:52,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:52,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:52,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:52,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634872792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634872792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:52,842 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:52,845 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411263aa18b61fad54797a54f2e7166430ad1_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263aa18b61fad54797a54f2e7166430ad1_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:52,846 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/a01fa5b5392e4b028977435fd9babbab, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:52,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/a01fa5b5392e4b028977435fd9babbab is 175, key is test_row_0/A:col10/1732634812427/Put/seqid=0 2024-11-26T15:26:52,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742233_1409 (size=39949) 2024-11-26T15:26:53,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:53,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634873113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:53,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:53,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634873113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:53,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:53,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:53,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:53,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:53,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:53,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:53,255 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/a01fa5b5392e4b028977435fd9babbab 2024-11-26T15:26:53,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/33feb948cd924ab299c417cbef97a473 is 50, key is test_row_0/B:col10/1732634812427/Put/seqid=0 2024-11-26T15:26:53,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742234_1410 (size=12301) 2024-11-26T15:26:53,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634873622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:53,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634873622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:53,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/33feb948cd924ab299c417cbef97a473 2024-11-26T15:26:53,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/3330864927424b858aa4bc766be5926f is 50, key is test_row_0/C:col10/1732634812427/Put/seqid=0 2024-11-26T15:26:53,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742235_1411 (size=12301) 2024-11-26T15:26:53,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/3330864927424b858aa4bc766be5926f 2024-11-26T15:26:53,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/a01fa5b5392e4b028977435fd9babbab as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a01fa5b5392e4b028977435fd9babbab 2024-11-26T15:26:53,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a01fa5b5392e4b028977435fd9babbab, entries=200, sequenceid=267, filesize=39.0 K 2024-11-26T15:26:53,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/33feb948cd924ab299c417cbef97a473 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/33feb948cd924ab299c417cbef97a473 2024-11-26T15:26:53,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/33feb948cd924ab299c417cbef97a473, entries=150, sequenceid=267, filesize=12.0 K 2024-11-26T15:26:53,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/3330864927424b858aa4bc766be5926f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3330864927424b858aa4bc766be5926f 2024-11-26T15:26:53,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3330864927424b858aa4bc766be5926f, entries=150, sequenceid=267, filesize=12.0 K 2024-11-26T15:26:53,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for ef027b3b2a337d7b7f5ab878c67d25f8 in 1302ms, sequenceid=267, compaction requested=true 2024-11-26T15:26:53,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:53,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef027b3b2a337d7b7f5ab878c67d25f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:26:53,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:53,730 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:26:53,730 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:26:53,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef027b3b2a337d7b7f5ab878c67d25f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:26:53,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:53,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef027b3b2a337d7b7f5ab878c67d25f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:26:53,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:53,732 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142352 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:26:53,732 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:26:53,732 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): ef027b3b2a337d7b7f5ab878c67d25f8/A is initiating minor compaction (all files) 2024-11-26T15:26:53,732 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ef027b3b2a337d7b7f5ab878c67d25f8/B is initiating minor compaction (all files) 2024-11-26T15:26:53,732 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef027b3b2a337d7b7f5ab878c67d25f8/B in TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:53,732 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef027b3b2a337d7b7f5ab878c67d25f8/A in TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:53,732 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/8057edaa647b4a0db4b30906d395a4fc, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/ca84c4954abf40a59feb94bb30f893bc, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/2aa073bf6b87447e90ad38634766d667, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/33feb948cd924ab299c417cbef97a473] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp, totalSize=48.0 K 2024-11-26T15:26:53,732 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/3d511b2afd024856b1eebcaefbc48efa, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/1071f1bcb87549b5be26d7ca417b4731, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/09f31c336cfc4656886987d46b745e95, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a01fa5b5392e4b028977435fd9babbab] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp, totalSize=139.0 K 2024-11-26T15:26:53,732 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:53,732 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/3d511b2afd024856b1eebcaefbc48efa, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/1071f1bcb87549b5be26d7ca417b4731, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/09f31c336cfc4656886987d46b745e95, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a01fa5b5392e4b028977435fd9babbab] 2024-11-26T15:26:53,732 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d511b2afd024856b1eebcaefbc48efa, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732634809434 2024-11-26T15:26:53,732 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 8057edaa647b4a0db4b30906d395a4fc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732634809434 2024-11-26T15:26:53,733 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting ca84c4954abf40a59feb94bb30f893bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732634809876 2024-11-26T15:26:53,733 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1071f1bcb87549b5be26d7ca417b4731, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732634809876 2024-11-26T15:26:53,733 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aa073bf6b87447e90ad38634766d667, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732634810050 2024-11-26T15:26:53,733 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09f31c336cfc4656886987d46b745e95, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732634810050 2024-11-26T15:26:53,734 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 33feb948cd924ab299c417cbef97a473, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732634812296 2024-11-26T15:26:53,734 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting a01fa5b5392e4b028977435fd9babbab, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732634812296 2024-11-26T15:26:53,740 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:53,743 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef027b3b2a337d7b7f5ab878c67d25f8#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:53,743 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/4bf451b3f73f43f3b796d00305e9722e is 50, key is test_row_0/B:col10/1732634812427/Put/seqid=0 2024-11-26T15:26:53,749 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126f4d1bd71407041e097765ffddf9d7add_ef027b3b2a337d7b7f5ab878c67d25f8 store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:53,752 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126f4d1bd71407041e097765ffddf9d7add_ef027b3b2a337d7b7f5ab878c67d25f8, store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:53,752 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f4d1bd71407041e097765ffddf9d7add_ef027b3b2a337d7b7f5ab878c67d25f8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:53,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742236_1412 (size=12881) 2024-11-26T15:26:53,774 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/4bf451b3f73f43f3b796d00305e9722e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/4bf451b3f73f43f3b796d00305e9722e 2024-11-26T15:26:53,781 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ef027b3b2a337d7b7f5ab878c67d25f8/B of ef027b3b2a337d7b7f5ab878c67d25f8 into 4bf451b3f73f43f3b796d00305e9722e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:53,781 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:53,781 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., storeName=ef027b3b2a337d7b7f5ab878c67d25f8/B, priority=12, startTime=1732634813730; duration=0sec 2024-11-26T15:26:53,781 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:53,782 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef027b3b2a337d7b7f5ab878c67d25f8:B 2024-11-26T15:26:53,782 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:26:53,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742237_1413 (size=4469) 2024-11-26T15:26:53,783 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:26:53,784 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ef027b3b2a337d7b7f5ab878c67d25f8/C is initiating minor compaction (all files) 2024-11-26T15:26:53,784 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef027b3b2a337d7b7f5ab878c67d25f8/C in TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:53,784 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/487e081861104f479bf40c10ec8c1382, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/ef4b2e1396864a24b500da7938b7af4f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/586028946bd84acb9e893ec91a9ea6f8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3330864927424b858aa4bc766be5926f] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp, totalSize=48.0 K 2024-11-26T15:26:53,784 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 487e081861104f479bf40c10ec8c1382, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732634809434 2024-11-26T15:26:53,784 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting ef4b2e1396864a24b500da7938b7af4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732634809876 2024-11-26T15:26:53,784 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef027b3b2a337d7b7f5ab878c67d25f8#A#compaction#344 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:53,785 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/4a72e5d88da7479e999be858ca92a286 is 175, key is test_row_0/A:col10/1732634812427/Put/seqid=0 2024-11-26T15:26:53,785 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 586028946bd84acb9e893ec91a9ea6f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732634810050 2024-11-26T15:26:53,786 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 3330864927424b858aa4bc766be5926f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732634812296 2024-11-26T15:26:53,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742238_1414 (size=31835) 2024-11-26T15:26:53,805 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/4a72e5d88da7479e999be858ca92a286 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4a72e5d88da7479e999be858ca92a286 2024-11-26T15:26:53,807 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef027b3b2a337d7b7f5ab878c67d25f8#C#compaction#346 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:53,808 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/a1c7030e52b54ab48e4bf323fa3d2f37 is 50, key is test_row_0/C:col10/1732634812427/Put/seqid=0 2024-11-26T15:26:53,812 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ef027b3b2a337d7b7f5ab878c67d25f8/A of ef027b3b2a337d7b7f5ab878c67d25f8 into 4a72e5d88da7479e999be858ca92a286(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:53,812 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:53,812 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., storeName=ef027b3b2a337d7b7f5ab878c67d25f8/A, priority=12, startTime=1732634813730; duration=0sec 2024-11-26T15:26:53,813 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:53,813 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef027b3b2a337d7b7f5ab878c67d25f8:A 2024-11-26T15:26:53,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742239_1415 (size=12881) 2024-11-26T15:26:54,230 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/a1c7030e52b54ab48e4bf323fa3d2f37 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/a1c7030e52b54ab48e4bf323fa3d2f37 2024-11-26T15:26:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-26T15:26:54,233 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:54,234 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:26:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-26T15:26:54,236 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ef027b3b2a337d7b7f5ab878c67d25f8/C of ef027b3b2a337d7b7f5ab878c67d25f8 into a1c7030e52b54ab48e4bf323fa3d2f37(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:54,236 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:54,236 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:26:54,236 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., storeName=ef027b3b2a337d7b7f5ab878c67d25f8/C, priority=12, startTime=1732634813730; duration=0sec 2024-11-26T15:26:54,236 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:54,237 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef027b3b2a337d7b7f5ab878c67d25f8:C 2024-11-26T15:26:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T15:26:54,237 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:26:54,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:26:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T15:26:54,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T15:26:54,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:54,390 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-26T15:26:54,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c8525aa6c74a460d8986efc6f49b59ab_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634812458/Put/seqid=0 2024-11-26T15:26:54,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742240_1416 (size=12454) 2024-11-26T15:26:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T15:26:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:54,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. as already flushing 2024-11-26T15:26:54,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634874695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:54,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:54,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:54,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:54,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634874698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:54,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634874802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:54,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:54,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:54,815 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c8525aa6c74a460d8986efc6f49b59ab_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c8525aa6c74a460d8986efc6f49b59ab_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:54,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/300d7c2982cb444191a07796abdf6476, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:54,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/300d7c2982cb444191a07796abdf6476 is 175, key is test_row_0/A:col10/1732634812458/Put/seqid=0 2024-11-26T15:26:54,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634874812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:54,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742241_1417 (size=31255) 2024-11-26T15:26:54,824 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/300d7c2982cb444191a07796abdf6476 2024-11-26T15:26:54,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/aa60a2f40d3043b9ac7490f34c9d68db is 50, key is test_row_0/B:col10/1732634812458/Put/seqid=0 2024-11-26T15:26:54,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742242_1418 (size=12301) 2024-11-26T15:26:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T15:26:55,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634875013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:55,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634875022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:55,235 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/aa60a2f40d3043b9ac7490f34c9d68db 2024-11-26T15:26:55,238 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T15:26:55,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/c1408b0a224743c88adc8ac873284bbd is 50, key is test_row_0/C:col10/1732634812458/Put/seqid=0 2024-11-26T15:26:55,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742243_1419 (size=12301) 2024-11-26T15:26:55,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634875332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:55,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634875332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:55,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T15:26:55,670 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/c1408b0a224743c88adc8ac873284bbd 2024-11-26T15:26:55,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/300d7c2982cb444191a07796abdf6476 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/300d7c2982cb444191a07796abdf6476 2024-11-26T15:26:55,678 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/300d7c2982cb444191a07796abdf6476, entries=150, sequenceid=282, filesize=30.5 K 2024-11-26T15:26:55,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/aa60a2f40d3043b9ac7490f34c9d68db as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/aa60a2f40d3043b9ac7490f34c9d68db 2024-11-26T15:26:55,683 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/aa60a2f40d3043b9ac7490f34c9d68db, entries=150, sequenceid=282, filesize=12.0 K 2024-11-26T15:26:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/c1408b0a224743c88adc8ac873284bbd as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/c1408b0a224743c88adc8ac873284bbd 2024-11-26T15:26:55,687 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/c1408b0a224743c88adc8ac873284bbd, entries=150, sequenceid=282, filesize=12.0 K 2024-11-26T15:26:55,688 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for ef027b3b2a337d7b7f5ab878c67d25f8 in 1297ms, sequenceid=282, compaction requested=false 2024-11-26T15:26:55,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:55,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:55,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-26T15:26:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-26T15:26:55,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-26T15:26:55,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4510 sec 2024-11-26T15:26:55,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.4560 sec 2024-11-26T15:26:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:55,855 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:55,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-26T15:26:55,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265e74a5877d2e47c38980bae943a86737_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634815854/Put/seqid=0 2024-11-26T15:26:55,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742244_1420 (size=14994) 2024-11-26T15:26:55,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634875876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:55,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:55,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:55,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:55,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634875880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:55,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634875982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:56,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634875993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:56,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:56,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:56,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:56,030 INFO [Thread-1571 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1571. 2024-11-26T15:26:56,031 DEBUG [Thread-1571 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:56,031 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:56,031 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:56,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:56,033 INFO [Thread-1577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1577. 2024-11-26T15:26:56,034 DEBUG [Thread-1577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:56,034 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:56,034 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:56,034 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:56,034 INFO [Thread-1573 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1573. 2024-11-26T15:26:56,034 DEBUG [Thread-1573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:56,034 DEBUG [Thread-1573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:56,034 DEBUG [Thread-1573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:56,035 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:56,042 INFO [Thread-1575 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1575. 2024-11-26T15:26:56,042 DEBUG [Thread-1575 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:56,042 DEBUG [Thread-1575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:56,042 DEBUG [Thread-1575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:56,042 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:56,043 INFO [Thread-1579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1579. 2024-11-26T15:26:56,043 DEBUG [Thread-1579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:56,043 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:56,043 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:56,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:56,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634876203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:56,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:56,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:56,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:56,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634876213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:56,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:56,266 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:56,271 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265e74a5877d2e47c38980bae943a86737_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265e74a5877d2e47c38980bae943a86737_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:56,272 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/4e71317a5a924fdd940230f0a0bc18da, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:56,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/4e71317a5a924fdd940230f0a0bc18da is 175, key is test_row_0/A:col10/1732634815854/Put/seqid=0 2024-11-26T15:26:56,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742245_1421 (size=39949) 2024-11-26T15:26:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T15:26:56,384 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:26:56,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634876513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:56,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634876522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:56,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:56,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:56,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:56,678 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/4e71317a5a924fdd940230f0a0bc18da 2024-11-26T15:26:56,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/2cc162bcd28e47589367f6a38be3565b is 50, key is test_row_0/B:col10/1732634815854/Put/seqid=0 2024-11-26T15:26:56,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742246_1422 (size=12301) 2024-11-26T15:26:57,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53800 deadline: 1732634877023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:57,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:57,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:26:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53786 deadline: 1732634877033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:26:57,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:57,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:26:57,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:26:57,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/2cc162bcd28e47589367f6a38be3565b 2024-11-26T15:26:57,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/218e2a13cf504b6db4d75e623683ae1a is 50, key is test_row_0/C:col10/1732634815854/Put/seqid=0 2024-11-26T15:26:57,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742247_1423 (size=12301) 2024-11-26T15:26:57,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/218e2a13cf504b6db4d75e623683ae1a 2024-11-26T15:26:57,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/4e71317a5a924fdd940230f0a0bc18da as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4e71317a5a924fdd940230f0a0bc18da 2024-11-26T15:26:57,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4e71317a5a924fdd940230f0a0bc18da, entries=200, sequenceid=307, filesize=39.0 K 2024-11-26T15:26:57,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/2cc162bcd28e47589367f6a38be3565b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/2cc162bcd28e47589367f6a38be3565b 2024-11-26T15:26:57,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/2cc162bcd28e47589367f6a38be3565b, entries=150, sequenceid=307, filesize=12.0 K 2024-11-26T15:26:57,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/218e2a13cf504b6db4d75e623683ae1a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/218e2a13cf504b6db4d75e623683ae1a 2024-11-26T15:26:57,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/218e2a13cf504b6db4d75e623683ae1a, entries=150, sequenceid=307, filesize=12.0 K 2024-11-26T15:26:57,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ef027b3b2a337d7b7f5ab878c67d25f8 in 1678ms, sequenceid=307, compaction requested=true 2024-11-26T15:26:57,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:57,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef027b3b2a337d7b7f5ab878c67d25f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:26:57,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:57,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef027b3b2a337d7b7f5ab878c67d25f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:26:57,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:57,534 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:57,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef027b3b2a337d7b7f5ab878c67d25f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:26:57,534 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:57,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:57,534 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:57,534 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:57,535 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): ef027b3b2a337d7b7f5ab878c67d25f8/A is initiating minor compaction (all files) 2024-11-26T15:26:57,535 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ef027b3b2a337d7b7f5ab878c67d25f8/B is initiating minor compaction (all files) 2024-11-26T15:26:57,535 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef027b3b2a337d7b7f5ab878c67d25f8/A in TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:57,535 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef027b3b2a337d7b7f5ab878c67d25f8/B in TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:57,535 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/4bf451b3f73f43f3b796d00305e9722e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/aa60a2f40d3043b9ac7490f34c9d68db, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/2cc162bcd28e47589367f6a38be3565b] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp, totalSize=36.6 K 2024-11-26T15:26:57,535 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4a72e5d88da7479e999be858ca92a286, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/300d7c2982cb444191a07796abdf6476, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4e71317a5a924fdd940230f0a0bc18da] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp, totalSize=100.6 K 2024-11-26T15:26:57,535 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:57,535 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4a72e5d88da7479e999be858ca92a286, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/300d7c2982cb444191a07796abdf6476, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4e71317a5a924fdd940230f0a0bc18da] 2024-11-26T15:26:57,535 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bf451b3f73f43f3b796d00305e9722e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732634812296 2024-11-26T15:26:57,535 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a72e5d88da7479e999be858ca92a286, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732634812296 2024-11-26T15:26:57,535 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting aa60a2f40d3043b9ac7490f34c9d68db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732634812457 2024-11-26T15:26:57,535 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 300d7c2982cb444191a07796abdf6476, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732634812457 2024-11-26T15:26:57,536 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cc162bcd28e47589367f6a38be3565b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732634814687 2024-11-26T15:26:57,536 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e71317a5a924fdd940230f0a0bc18da, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732634814687 2024-11-26T15:26:57,545 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef027b3b2a337d7b7f5ab878c67d25f8#B#compaction#353 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:57,545 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/6a38d6a567b34bad8f1c0e56da1758f3 is 50, key is test_row_0/B:col10/1732634815854/Put/seqid=0 2024-11-26T15:26:57,546 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:57,547 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411261da50b87eb634965bc2a44e3321dcaa4_ef027b3b2a337d7b7f5ab878c67d25f8 store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:57,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742248_1424 (size=12983) 2024-11-26T15:26:57,580 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411261da50b87eb634965bc2a44e3321dcaa4_ef027b3b2a337d7b7f5ab878c67d25f8, store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:57,581 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261da50b87eb634965bc2a44e3321dcaa4_ef027b3b2a337d7b7f5ab878c67d25f8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:57,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742249_1425 (size=4469) 2024-11-26T15:26:57,957 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/6a38d6a567b34bad8f1c0e56da1758f3 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/6a38d6a567b34bad8f1c0e56da1758f3 2024-11-26T15:26:57,964 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef027b3b2a337d7b7f5ab878c67d25f8/B of ef027b3b2a337d7b7f5ab878c67d25f8 into 6a38d6a567b34bad8f1c0e56da1758f3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:57,964 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:57,964 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., storeName=ef027b3b2a337d7b7f5ab878c67d25f8/B, priority=13, startTime=1732634817533; duration=0sec 2024-11-26T15:26:57,964 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:26:57,964 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef027b3b2a337d7b7f5ab878c67d25f8:B 2024-11-26T15:26:57,964 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:26:57,965 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:26:57,965 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ef027b3b2a337d7b7f5ab878c67d25f8/C is initiating minor compaction (all files) 2024-11-26T15:26:57,965 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef027b3b2a337d7b7f5ab878c67d25f8/C in TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:26:57,965 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/a1c7030e52b54ab48e4bf323fa3d2f37, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/c1408b0a224743c88adc8ac873284bbd, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/218e2a13cf504b6db4d75e623683ae1a] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp, totalSize=36.6 K 2024-11-26T15:26:57,965 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting a1c7030e52b54ab48e4bf323fa3d2f37, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732634812296 2024-11-26T15:26:57,966 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting c1408b0a224743c88adc8ac873284bbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732634812457 2024-11-26T15:26:57,966 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 218e2a13cf504b6db4d75e623683ae1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732634814687 2024-11-26T15:26:57,971 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef027b3b2a337d7b7f5ab878c67d25f8#C#compaction#355 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:57,971 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/6dc6d9effcae4d08af9487fae0a34b79 is 50, key is test_row_0/C:col10/1732634815854/Put/seqid=0 2024-11-26T15:26:57,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742250_1426 (size=12983) 2024-11-26T15:26:57,985 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef027b3b2a337d7b7f5ab878c67d25f8#A#compaction#354 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-26T15:26:57,986 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/f8a2947bc864462b92a018b14b4554e0 is 175, key is test_row_0/A:col10/1732634815854/Put/seqid=0 2024-11-26T15:26:57,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742251_1427 (size=31937) 2024-11-26T15:26:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:58,045 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:26:58,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-26T15:26:58,045 INFO [Thread-1564 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1564. 2024-11-26T15:26:58,045 INFO [Thread-1560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1560. 2024-11-26T15:26:58,045 DEBUG [Thread-1560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:58,045 DEBUG [Thread-1564 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:26:58,045 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:58,045 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:58,045 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:58,045 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:26:58,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:58,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:26:58,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126dc2bdccb7bbb4d329bf63b4efbe81c8d_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634815872/Put/seqid=0 2024-11-26T15:26:58,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742252_1428 (size=12454) 2024-11-26T15:26:58,380 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/6dc6d9effcae4d08af9487fae0a34b79 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/6dc6d9effcae4d08af9487fae0a34b79 2024-11-26T15:26:58,387 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef027b3b2a337d7b7f5ab878c67d25f8/C of ef027b3b2a337d7b7f5ab878c67d25f8 into 6dc6d9effcae4d08af9487fae0a34b79(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:58,387 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:58,387 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., storeName=ef027b3b2a337d7b7f5ab878c67d25f8/C, priority=13, startTime=1732634817534; duration=0sec 2024-11-26T15:26:58,387 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:58,387 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef027b3b2a337d7b7f5ab878c67d25f8:C 2024-11-26T15:26:58,394 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/f8a2947bc864462b92a018b14b4554e0 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/f8a2947bc864462b92a018b14b4554e0 2024-11-26T15:26:58,398 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef027b3b2a337d7b7f5ab878c67d25f8/A of ef027b3b2a337d7b7f5ab878c67d25f8 into f8a2947bc864462b92a018b14b4554e0(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:26:58,398 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:26:58,398 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., storeName=ef027b3b2a337d7b7f5ab878c67d25f8/A, priority=13, startTime=1732634817533; duration=0sec 2024-11-26T15:26:58,398 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:26:58,398 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef027b3b2a337d7b7f5ab878c67d25f8:A 2024-11-26T15:26:58,459 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:26:58,465 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126dc2bdccb7bbb4d329bf63b4efbe81c8d_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126dc2bdccb7bbb4d329bf63b4efbe81c8d_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:26:58,467 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/4613c1807e0a4c7e9b6adaec9a48e23c, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:26:58,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/4613c1807e0a4c7e9b6adaec9a48e23c is 175, key is test_row_0/A:col10/1732634815872/Put/seqid=0 2024-11-26T15:26:58,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742253_1429 (size=31255) 2024-11-26T15:26:58,875 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/4613c1807e0a4c7e9b6adaec9a48e23c 2024-11-26T15:26:58,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/94410a9991cd4561ae2da42cfb763d03 is 50, key is test_row_0/B:col10/1732634815872/Put/seqid=0 2024-11-26T15:26:58,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742254_1430 (size=12301) 2024-11-26T15:26:59,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/94410a9991cd4561ae2da42cfb763d03 2024-11-26T15:26:59,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/363230897cae481185f36dbe601b8239 is 50, key is test_row_0/C:col10/1732634815872/Put/seqid=0 2024-11-26T15:26:59,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742255_1431 (size=12301) 2024-11-26T15:26:59,581 DEBUG [regionserver/6cc2bc8bf733:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-11-26T15:26:59,587 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/5c0a2828eb334c8894434a262680d3a9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a27c1eefeddf4520aa0420183801682c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/bca6d2aaac5b44b7b2d52e331261b0fd, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/73fbf91aa56f47c997a5106e7c45ae04, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/9b52c9092a564b5ea10c92a7347f890f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4d08254c63ac42d4a999265ba381edec, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/de95275812ba4a1b99941c02e3528f5a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/3dcd0eceab5c4663b75270da14254578, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/d3487054f2ef479ab31dc4e3c45655b1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/27033cf6211f47ae8bc3c31c82f3c05e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/7586f9633a6a4d0a8fc02de12436eada, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a02d0354ae29469c87baa6df95396665, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/655cafc6326b4ff4844b1e1d2ee6f26e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/3d511b2afd024856b1eebcaefbc48efa, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/1071f1bcb87549b5be26d7ca417b4731, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/09f31c336cfc4656886987d46b745e95, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a01fa5b5392e4b028977435fd9babbab, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4a72e5d88da7479e999be858ca92a286, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/300d7c2982cb444191a07796abdf6476, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4e71317a5a924fdd940230f0a0bc18da] to archive 2024-11-26T15:26:59,589 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:26:59,592 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/5c0a2828eb334c8894434a262680d3a9 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/5c0a2828eb334c8894434a262680d3a9 2024-11-26T15:26:59,593 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a27c1eefeddf4520aa0420183801682c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a27c1eefeddf4520aa0420183801682c 2024-11-26T15:26:59,594 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/bca6d2aaac5b44b7b2d52e331261b0fd to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/bca6d2aaac5b44b7b2d52e331261b0fd 2024-11-26T15:26:59,595 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/73fbf91aa56f47c997a5106e7c45ae04 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/73fbf91aa56f47c997a5106e7c45ae04 2024-11-26T15:26:59,596 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/9b52c9092a564b5ea10c92a7347f890f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/9b52c9092a564b5ea10c92a7347f890f 2024-11-26T15:26:59,596 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4d08254c63ac42d4a999265ba381edec to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4d08254c63ac42d4a999265ba381edec 2024-11-26T15:26:59,597 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/de95275812ba4a1b99941c02e3528f5a to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/de95275812ba4a1b99941c02e3528f5a 2024-11-26T15:26:59,598 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/3dcd0eceab5c4663b75270da14254578 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/3dcd0eceab5c4663b75270da14254578 2024-11-26T15:26:59,599 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/d3487054f2ef479ab31dc4e3c45655b1 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/d3487054f2ef479ab31dc4e3c45655b1 2024-11-26T15:26:59,599 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/27033cf6211f47ae8bc3c31c82f3c05e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/27033cf6211f47ae8bc3c31c82f3c05e 2024-11-26T15:26:59,600 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/7586f9633a6a4d0a8fc02de12436eada to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/7586f9633a6a4d0a8fc02de12436eada 2024-11-26T15:26:59,601 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a02d0354ae29469c87baa6df95396665 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a02d0354ae29469c87baa6df95396665 2024-11-26T15:26:59,601 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/655cafc6326b4ff4844b1e1d2ee6f26e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/655cafc6326b4ff4844b1e1d2ee6f26e 2024-11-26T15:26:59,602 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/3d511b2afd024856b1eebcaefbc48efa to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/3d511b2afd024856b1eebcaefbc48efa 2024-11-26T15:26:59,603 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/1071f1bcb87549b5be26d7ca417b4731 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/1071f1bcb87549b5be26d7ca417b4731 2024-11-26T15:26:59,603 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/09f31c336cfc4656886987d46b745e95 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/09f31c336cfc4656886987d46b745e95 2024-11-26T15:26:59,604 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a01fa5b5392e4b028977435fd9babbab to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/a01fa5b5392e4b028977435fd9babbab 2024-11-26T15:26:59,605 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4a72e5d88da7479e999be858ca92a286 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4a72e5d88da7479e999be858ca92a286 2024-11-26T15:26:59,606 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/300d7c2982cb444191a07796abdf6476 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/300d7c2982cb444191a07796abdf6476 2024-11-26T15:26:59,607 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4e71317a5a924fdd940230f0a0bc18da to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4e71317a5a924fdd940230f0a0bc18da 2024-11-26T15:26:59,610 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/267854020ddc461bae9382b52ba088dd, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/96be9cabf86b446282c524b3e8b9b2e5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/80ab99748e6e47d4a4dc03c862f3d6d9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/1b3f45d976834053bb8808220b78cd2b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/c9a61454da60497c91475d390228c864, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/d7a8222c250442ffb7e26300a2eca345, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/5350372298f14bac9bc38928693e617d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/dc51efb5e075480980c7e9563676b1bf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/ad65bc43bd0c4ada90b18eedbed92520, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/7c73a1508cab412c964d193468350cad, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/45ecdfc2816947d79bb8d68302534b8a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/43ecbaaf27594b3daa8a87c2d554a070, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/8057edaa647b4a0db4b30906d395a4fc, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/e6f5802ac82441c799ab4afc96d265d9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/ca84c4954abf40a59feb94bb30f893bc, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/2aa073bf6b87447e90ad38634766d667, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/4bf451b3f73f43f3b796d00305e9722e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/33feb948cd924ab299c417cbef97a473, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/aa60a2f40d3043b9ac7490f34c9d68db, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/2cc162bcd28e47589367f6a38be3565b] to archive 2024-11-26T15:26:59,610 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:26:59,611 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/267854020ddc461bae9382b52ba088dd to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/267854020ddc461bae9382b52ba088dd 2024-11-26T15:26:59,612 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/96be9cabf86b446282c524b3e8b9b2e5 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/96be9cabf86b446282c524b3e8b9b2e5 2024-11-26T15:26:59,613 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/80ab99748e6e47d4a4dc03c862f3d6d9 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/80ab99748e6e47d4a4dc03c862f3d6d9 2024-11-26T15:26:59,614 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/1b3f45d976834053bb8808220b78cd2b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/1b3f45d976834053bb8808220b78cd2b 2024-11-26T15:26:59,615 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/c9a61454da60497c91475d390228c864 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/c9a61454da60497c91475d390228c864 2024-11-26T15:26:59,615 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/d7a8222c250442ffb7e26300a2eca345 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/d7a8222c250442ffb7e26300a2eca345 2024-11-26T15:26:59,616 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/5350372298f14bac9bc38928693e617d to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/5350372298f14bac9bc38928693e617d 2024-11-26T15:26:59,617 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/dc51efb5e075480980c7e9563676b1bf to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/dc51efb5e075480980c7e9563676b1bf 2024-11-26T15:26:59,618 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/ad65bc43bd0c4ada90b18eedbed92520 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/ad65bc43bd0c4ada90b18eedbed92520 2024-11-26T15:26:59,618 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/7c73a1508cab412c964d193468350cad to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/7c73a1508cab412c964d193468350cad 2024-11-26T15:26:59,619 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/45ecdfc2816947d79bb8d68302534b8a to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/45ecdfc2816947d79bb8d68302534b8a 2024-11-26T15:26:59,619 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/43ecbaaf27594b3daa8a87c2d554a070 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/43ecbaaf27594b3daa8a87c2d554a070 2024-11-26T15:26:59,620 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/8057edaa647b4a0db4b30906d395a4fc to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/8057edaa647b4a0db4b30906d395a4fc 2024-11-26T15:26:59,621 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/e6f5802ac82441c799ab4afc96d265d9 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/e6f5802ac82441c799ab4afc96d265d9 2024-11-26T15:26:59,621 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/ca84c4954abf40a59feb94bb30f893bc to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/ca84c4954abf40a59feb94bb30f893bc 2024-11-26T15:26:59,622 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/2aa073bf6b87447e90ad38634766d667 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/2aa073bf6b87447e90ad38634766d667 2024-11-26T15:26:59,623 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/4bf451b3f73f43f3b796d00305e9722e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/4bf451b3f73f43f3b796d00305e9722e 2024-11-26T15:26:59,623 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/33feb948cd924ab299c417cbef97a473 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/33feb948cd924ab299c417cbef97a473 2024-11-26T15:26:59,624 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/aa60a2f40d3043b9ac7490f34c9d68db to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/aa60a2f40d3043b9ac7490f34c9d68db 2024-11-26T15:26:59,625 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/2cc162bcd28e47589367f6a38be3565b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/2cc162bcd28e47589367f6a38be3565b 2024-11-26T15:26:59,627 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/9e89209d90a04467a2cfa4ccf516bb29, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/821d7cdd62d2474183358ab803b785be, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/89bdfe3150a1400f83e0485025580a5a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/e8a56155e2ba4296b70267e5ceb42347, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/d7809ed92b504d43b1459ad807788569, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3062de8224454b49846c0df66b0a27ca, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/2e3b3fa4d44940dfabf29a3a16a28fa5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/925af371837848649b3daa192e69cfa8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/5f8cdd18e56948b290a1a591c9460073, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3bb63b491b914e11a0910d3b01771d6e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/9caf73cdc9684c95b62faf52b17f6e36, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/1c13f7efd87f484687dba7278c750edb, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/487e081861104f479bf40c10ec8c1382, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/65477bc3499f4a08bf2837556cea32a7, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/ef4b2e1396864a24b500da7938b7af4f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/586028946bd84acb9e893ec91a9ea6f8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/a1c7030e52b54ab48e4bf323fa3d2f37, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3330864927424b858aa4bc766be5926f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/c1408b0a224743c88adc8ac873284bbd, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/218e2a13cf504b6db4d75e623683ae1a] to archive 2024-11-26T15:26:59,628 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:26:59,629 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/9e89209d90a04467a2cfa4ccf516bb29 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/9e89209d90a04467a2cfa4ccf516bb29 2024-11-26T15:26:59,630 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/821d7cdd62d2474183358ab803b785be to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/821d7cdd62d2474183358ab803b785be 2024-11-26T15:26:59,631 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/89bdfe3150a1400f83e0485025580a5a to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/89bdfe3150a1400f83e0485025580a5a 2024-11-26T15:26:59,631 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/e8a56155e2ba4296b70267e5ceb42347 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/e8a56155e2ba4296b70267e5ceb42347 2024-11-26T15:26:59,632 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/d7809ed92b504d43b1459ad807788569 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/d7809ed92b504d43b1459ad807788569 2024-11-26T15:26:59,633 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3062de8224454b49846c0df66b0a27ca to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3062de8224454b49846c0df66b0a27ca 2024-11-26T15:26:59,633 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/2e3b3fa4d44940dfabf29a3a16a28fa5 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/2e3b3fa4d44940dfabf29a3a16a28fa5 2024-11-26T15:26:59,634 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/925af371837848649b3daa192e69cfa8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/925af371837848649b3daa192e69cfa8 2024-11-26T15:26:59,635 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/5f8cdd18e56948b290a1a591c9460073 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/5f8cdd18e56948b290a1a591c9460073 2024-11-26T15:26:59,636 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3bb63b491b914e11a0910d3b01771d6e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3bb63b491b914e11a0910d3b01771d6e 2024-11-26T15:26:59,636 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/9caf73cdc9684c95b62faf52b17f6e36 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/9caf73cdc9684c95b62faf52b17f6e36 2024-11-26T15:26:59,637 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/1c13f7efd87f484687dba7278c750edb to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/1c13f7efd87f484687dba7278c750edb 2024-11-26T15:26:59,638 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/487e081861104f479bf40c10ec8c1382 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/487e081861104f479bf40c10ec8c1382 2024-11-26T15:26:59,639 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/65477bc3499f4a08bf2837556cea32a7 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/65477bc3499f4a08bf2837556cea32a7 2024-11-26T15:26:59,640 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/ef4b2e1396864a24b500da7938b7af4f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/ef4b2e1396864a24b500da7938b7af4f 2024-11-26T15:26:59,641 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/586028946bd84acb9e893ec91a9ea6f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/586028946bd84acb9e893ec91a9ea6f8 2024-11-26T15:26:59,642 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/a1c7030e52b54ab48e4bf323fa3d2f37 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/a1c7030e52b54ab48e4bf323fa3d2f37 2024-11-26T15:26:59,642 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3330864927424b858aa4bc766be5926f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/3330864927424b858aa4bc766be5926f 2024-11-26T15:26:59,643 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/c1408b0a224743c88adc8ac873284bbd to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/c1408b0a224743c88adc8ac873284bbd 2024-11-26T15:26:59,644 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/218e2a13cf504b6db4d75e623683ae1a to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/218e2a13cf504b6db4d75e623683ae1a 2024-11-26T15:26:59,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/363230897cae481185f36dbe601b8239 2024-11-26T15:26:59,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/4613c1807e0a4c7e9b6adaec9a48e23c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4613c1807e0a4c7e9b6adaec9a48e23c 2024-11-26T15:26:59,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4613c1807e0a4c7e9b6adaec9a48e23c, entries=150, sequenceid=321, filesize=30.5 K 2024-11-26T15:26:59,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/94410a9991cd4561ae2da42cfb763d03 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/94410a9991cd4561ae2da42cfb763d03 2024-11-26T15:26:59,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/94410a9991cd4561ae2da42cfb763d03, entries=150, sequenceid=321, filesize=12.0 K 2024-11-26T15:26:59,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/363230897cae481185f36dbe601b8239 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/363230897cae481185f36dbe601b8239 2024-11-26T15:26:59,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/363230897cae481185f36dbe601b8239, entries=150, sequenceid=321, filesize=12.0 K 2024-11-26T15:26:59,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for ef027b3b2a337d7b7f5ab878c67d25f8 in 1680ms, sequenceid=321, compaction requested=false 2024-11-26T15:26:59,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef027b3b2a337d7b7f5ab878c67d25f8: 2024-11-26T15:27:02,777 INFO [Thread-1562 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1562. 2024-11-26T15:27:02,778 DEBUG [Thread-1562 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:27:02,778 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:02,778 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:02,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:27:02,807 INFO [Thread-1568 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1568. 2024-11-26T15:27:02,807 DEBUG [Thread-1568 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:27:02,807 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:02,808 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:02,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:27:02,846 INFO [Thread-1566 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1566. 2024-11-26T15:27:02,846 DEBUG [Thread-1566 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:27:02,847 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:02,847 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:02,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:27:02,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T15:27:02,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-26T15:27:02,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-26T15:27:02,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-26T15:27:02,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-26T15:27:02,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-26T15:27:02,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T15:27:02,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T15:27:02,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1318 2024-11-26T15:27:02,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3954 rows 2024-11-26T15:27:02,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1316 2024-11-26T15:27:02,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3948 rows 2024-11-26T15:27:02,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1321 2024-11-26T15:27:02,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3961 rows 2024-11-26T15:27:02,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1318 2024-11-26T15:27:02,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3952 rows 2024-11-26T15:27:02,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1311 2024-11-26T15:27:02,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3933 rows 2024-11-26T15:27:02,848 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T15:27:02,849 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T15:27:02,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:02,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:02,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:27:02,851 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T15:27:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:02,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T15:27:02,856 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634822856"}]},"ts":"1732634822856"} 2024-11-26T15:27:02,858 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T15:27:02,858 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T15:27:02,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T15:27:02,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef027b3b2a337d7b7f5ab878c67d25f8, UNASSIGN}] 2024-11-26T15:27:02,861 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef027b3b2a337d7b7f5ab878c67d25f8, UNASSIGN 2024-11-26T15:27:02,862 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=ef027b3b2a337d7b7f5ab878c67d25f8, regionState=CLOSING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:02,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef027b3b2a337d7b7f5ab878c67d25f8, UNASSIGN because future has completed 2024-11-26T15:27:02,864 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T15:27:02,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:27:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T15:27:03,017 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:03,018 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T15:27:03,018 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing ef027b3b2a337d7b7f5ab878c67d25f8, disabling compactions & flushes 2024-11-26T15:27:03,018 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:27:03,018 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:27:03,018 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. after waiting 0 ms 2024-11-26T15:27:03,018 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:27:03,018 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing ef027b3b2a337d7b7f5ab878c67d25f8 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-26T15:27:03,029 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e6992538d8d942358d8c649a72529fe3_ef027b3b2a337d7b7f5ab878c67d25f8 is 50, key is test_row_0/A:col10/1732634822804/Put/seqid=0 2024-11-26T15:27:03,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742256_1432 (size=12454) 2024-11-26T15:27:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T15:27:03,433 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:03,441 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e6992538d8d942358d8c649a72529fe3_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e6992538d8d942358d8c649a72529fe3_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:03,443 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/ea27d107804d4005abc9df24026b9878, store: [table=TestAcidGuarantees family=A region=ef027b3b2a337d7b7f5ab878c67d25f8] 2024-11-26T15:27:03,444 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/ea27d107804d4005abc9df24026b9878 is 175, key is test_row_0/A:col10/1732634822804/Put/seqid=0 2024-11-26T15:27:03,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742257_1433 (size=31255) 2024-11-26T15:27:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T15:27:03,848 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/ea27d107804d4005abc9df24026b9878 2024-11-26T15:27:03,859 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/251afd60503e4c12b7638d9304eec938 is 50, key is test_row_0/B:col10/1732634822804/Put/seqid=0 2024-11-26T15:27:03,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742258_1434 (size=12301) 2024-11-26T15:27:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T15:27:04,263 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/251afd60503e4c12b7638d9304eec938 2024-11-26T15:27:04,274 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/8b10e55ac0b34c1fbd209619967287e1 is 50, key is test_row_0/C:col10/1732634822804/Put/seqid=0 2024-11-26T15:27:04,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742259_1435 (size=12301) 2024-11-26T15:27:04,678 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/8b10e55ac0b34c1fbd209619967287e1 2024-11-26T15:27:04,684 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/A/ea27d107804d4005abc9df24026b9878 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/ea27d107804d4005abc9df24026b9878 2024-11-26T15:27:04,689 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/ea27d107804d4005abc9df24026b9878, entries=150, sequenceid=329, filesize=30.5 K 2024-11-26T15:27:04,690 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/B/251afd60503e4c12b7638d9304eec938 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/251afd60503e4c12b7638d9304eec938 2024-11-26T15:27:04,693 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/251afd60503e4c12b7638d9304eec938, entries=150, sequenceid=329, filesize=12.0 K 2024-11-26T15:27:04,694 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/.tmp/C/8b10e55ac0b34c1fbd209619967287e1 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/8b10e55ac0b34c1fbd209619967287e1 2024-11-26T15:27:04,696 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/8b10e55ac0b34c1fbd209619967287e1, entries=150, sequenceid=329, filesize=12.0 K 2024-11-26T15:27:04,697 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for ef027b3b2a337d7b7f5ab878c67d25f8 in 1679ms, sequenceid=329, compaction requested=true 2024-11-26T15:27:04,701 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/recovered.edits/332.seqid, newMaxSeqId=332, maxSeqId=4 2024-11-26T15:27:04,701 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. 2024-11-26T15:27:04,701 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for ef027b3b2a337d7b7f5ab878c67d25f8: Waiting for close lock at 1732634823018Running coprocessor pre-close hooks at 1732634823018Disabling compacts and flushes for region at 1732634823018Disabling writes for close at 1732634823018Obtaining lock to block concurrent updates at 1732634823018Preparing flush snapshotting stores in ef027b3b2a337d7b7f5ab878c67d25f8 at 1732634823018Finished memstore snapshotting TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732634823019 (+1 ms)Flushing stores of TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8. at 1732634823020 (+1 ms)Flushing ef027b3b2a337d7b7f5ab878c67d25f8/A: creating writer at 1732634823020Flushing mob file ef027b3b2a337d7b7f5ab878c67d25f8/A: appending metadata at 1732634823028 (+8 ms)Flushing mob file ef027b3b2a337d7b7f5ab878c67d25f8/A: closing flushed file at 1732634823028Flushing ef027b3b2a337d7b7f5ab878c67d25f8/A: appending metadata at 1732634823443 (+415 ms)Flushing ef027b3b2a337d7b7f5ab878c67d25f8/A: closing flushed file at 1732634823443Flushing ef027b3b2a337d7b7f5ab878c67d25f8/B: creating writer at 1732634823856 (+413 ms)Flushing ef027b3b2a337d7b7f5ab878c67d25f8/B: appending metadata at 1732634823859 (+3 ms)Flushing ef027b3b2a337d7b7f5ab878c67d25f8/B: closing flushed file at 1732634823859Flushing ef027b3b2a337d7b7f5ab878c67d25f8/C: creating writer at 1732634824269 (+410 ms)Flushing ef027b3b2a337d7b7f5ab878c67d25f8/C: appending metadata at 1732634824273 (+4 ms)Flushing ef027b3b2a337d7b7f5ab878c67d25f8/C: closing flushed file at 1732634824273Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63c120d1: reopening flushed file at 1732634824683 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51c962a1: reopening flushed file at 1732634824689 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ffba137: reopening flushed file at 1732634824693 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for ef027b3b2a337d7b7f5ab878c67d25f8 in 1679ms, sequenceid=329, compaction requested=true at 1732634824697 (+4 ms)Writing region close event to WAL at 1732634824698 (+1 ms)Running coprocessor post-close hooks at 1732634824701 (+3 ms)Closed at 1732634824701 2024-11-26T15:27:04,703 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:04,704 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=ef027b3b2a337d7b7f5ab878c67d25f8, regionState=CLOSED 2024-11-26T15:27:04,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:27:04,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-26T15:27:04,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure ef027b3b2a337d7b7f5ab878c67d25f8, server=6cc2bc8bf733,45729,1732634697146 in 1.8420 sec 2024-11-26T15:27:04,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-26T15:27:04,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef027b3b2a337d7b7f5ab878c67d25f8, UNASSIGN in 1.8470 sec 2024-11-26T15:27:04,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-26T15:27:04,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8500 sec 2024-11-26T15:27:04,711 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634824711"}]},"ts":"1732634824711"} 2024-11-26T15:27:04,712 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T15:27:04,712 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T15:27:04,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8620 sec 2024-11-26T15:27:05,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T15:27:05,004 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:05,005 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T15:27:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:05,010 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T15:27:05,012 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:05,014 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,017 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/recovered.edits] 2024-11-26T15:27:05,019 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4613c1807e0a4c7e9b6adaec9a48e23c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/4613c1807e0a4c7e9b6adaec9a48e23c 2024-11-26T15:27:05,021 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/ea27d107804d4005abc9df24026b9878 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/ea27d107804d4005abc9df24026b9878 2024-11-26T15:27:05,022 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/f8a2947bc864462b92a018b14b4554e0 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/A/f8a2947bc864462b92a018b14b4554e0 2024-11-26T15:27:05,025 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/251afd60503e4c12b7638d9304eec938 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/251afd60503e4c12b7638d9304eec938 2024-11-26T15:27:05,026 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/6a38d6a567b34bad8f1c0e56da1758f3 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/6a38d6a567b34bad8f1c0e56da1758f3 2024-11-26T15:27:05,028 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/94410a9991cd4561ae2da42cfb763d03 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/B/94410a9991cd4561ae2da42cfb763d03 2024-11-26T15:27:05,030 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/363230897cae481185f36dbe601b8239 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/363230897cae481185f36dbe601b8239 2024-11-26T15:27:05,031 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/6dc6d9effcae4d08af9487fae0a34b79 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/6dc6d9effcae4d08af9487fae0a34b79 2024-11-26T15:27:05,032 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/8b10e55ac0b34c1fbd209619967287e1 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/C/8b10e55ac0b34c1fbd209619967287e1 2024-11-26T15:27:05,034 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/recovered.edits/332.seqid to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8/recovered.edits/332.seqid 2024-11-26T15:27:05,035 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,035 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T15:27:05,035 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T15:27:05,036 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T15:27:05,038 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126010b4ea3cde24f0ea1a3a9843f20eca2_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126010b4ea3cde24f0ea1a3a9843f20eca2_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,039 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263aa18b61fad54797a54f2e7166430ad1_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263aa18b61fad54797a54f2e7166430ad1_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,039 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263c79c008bbba41bf9070c46af1e78129_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411263c79c008bbba41bf9070c46af1e78129_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,040 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126510a670f38de42d0a78e940ef73f0337_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126510a670f38de42d0a78e940ef73f0337_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,041 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265e74a5877d2e47c38980bae943a86737_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265e74a5877d2e47c38980bae943a86737_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,042 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f2ff73c6e6749ee882c94387a1781f5_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265f2ff73c6e6749ee882c94387a1781f5_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,043 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267046b3d397ef4c7eace6813a6c906c16_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267046b3d397ef4c7eace6813a6c906c16_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,043 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269fc9bd1f571743489a61390f8f930401_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411269fc9bd1f571743489a61390f8f930401_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,044 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b50bd8b3d44b4517a6d2c9d5a6807ce8_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b50bd8b3d44b4517a6d2c9d5a6807ce8_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,045 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b9808e6b8e2b4e9cbb5a61d115b9879b_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b9808e6b8e2b4e9cbb5a61d115b9879b_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,046 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126be14611b9280449685aaa584272a21b1_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126be14611b9280449685aaa584272a21b1_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,047 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c4b6583316d1477daac6cf4a035757ff_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c4b6583316d1477daac6cf4a035757ff_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,048 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c8525aa6c74a460d8986efc6f49b59ab_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c8525aa6c74a460d8986efc6f49b59ab_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,049 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d6df1dc50c5541499cba42a3ad5aa1b7_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d6df1dc50c5541499cba42a3ad5aa1b7_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,049 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126dc2bdccb7bbb4d329bf63b4efbe81c8d_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126dc2bdccb7bbb4d329bf63b4efbe81c8d_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,050 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e0010603f7404b3f8fb76547ca167a1e_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e0010603f7404b3f8fb76547ca167a1e_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,051 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e6992538d8d942358d8c649a72529fe3_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e6992538d8d942358d8c649a72529fe3_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,052 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f96642ce4b5348b8aacc514674e27208_ef027b3b2a337d7b7f5ab878c67d25f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f96642ce4b5348b8aacc514674e27208_ef027b3b2a337d7b7f5ab878c67d25f8 2024-11-26T15:27:05,052 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T15:27:05,054 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:05,056 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T15:27:05,057 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T15:27:05,058 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:05,058 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T15:27:05,058 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732634825058"}]},"ts":"9223372036854775807"} 2024-11-26T15:27:05,060 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T15:27:05,060 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ef027b3b2a337d7b7f5ab878c67d25f8, NAME => 'TestAcidGuarantees,,1732634792869.ef027b3b2a337d7b7f5ab878c67d25f8.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T15:27:05,060 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T15:27:05,060 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732634825060"}]},"ts":"9223372036854775807"} 2024-11-26T15:27:05,062 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T15:27:05,062 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:05,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-11-26T15:27:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T15:27:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T15:27:05,123 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:05,135 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238 (was 236) - Thread LEAK? -, OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=461 (was 473), ProcessCount=11 (was 11), AvailableMemoryMB=9322 (was 9225) - AvailableMemoryMB LEAK? - 2024-11-26T15:27:05,143 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=461, ProcessCount=11, AvailableMemoryMB=9322 2024-11-26T15:27:05,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T15:27:05,145 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T15:27:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:05,147 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T15:27:05,147 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:05,148 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-26T15:27:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-26T15:27:05,149 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T15:27:05,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742260_1436 (size=959) 2024-11-26T15:27:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-26T15:27:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-26T15:27:05,558 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6 2024-11-26T15:27:05,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742261_1437 (size=53) 2024-11-26T15:27:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-26T15:27:05,967 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:27:05,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing af3054c22fab85f44f2e3f3ac4020be7, disabling compactions & flushes 2024-11-26T15:27:05,968 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:05,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:05,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. after waiting 0 ms 2024-11-26T15:27:05,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:05,968 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:05,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for af3054c22fab85f44f2e3f3ac4020be7: Waiting for close lock at 1732634825968Disabling compacts and flushes for region at 1732634825968Disabling writes for close at 1732634825968Writing region close event to WAL at 1732634825968Closed at 1732634825968 2024-11-26T15:27:05,970 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T15:27:05,970 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732634825970"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732634825970"}]},"ts":"1732634825970"} 2024-11-26T15:27:05,972 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T15:27:05,974 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T15:27:05,974 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634825974"}]},"ts":"1732634825974"} 2024-11-26T15:27:05,976 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T15:27:05,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=af3054c22fab85f44f2e3f3ac4020be7, ASSIGN}] 2024-11-26T15:27:05,977 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=af3054c22fab85f44f2e3f3ac4020be7, ASSIGN 2024-11-26T15:27:05,978 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=af3054c22fab85f44f2e3f3ac4020be7, ASSIGN; state=OFFLINE, location=6cc2bc8bf733,45729,1732634697146; forceNewPlan=false, retain=false 2024-11-26T15:27:06,129 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=af3054c22fab85f44f2e3f3ac4020be7, regionState=OPENING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:06,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=af3054c22fab85f44f2e3f3ac4020be7, ASSIGN because future has completed 2024-11-26T15:27:06,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:27:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-26T15:27:06,292 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:06,292 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:27:06,293 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:06,293 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:27:06,294 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:06,294 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:06,296 INFO [StoreOpener-af3054c22fab85f44f2e3f3ac4020be7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:06,298 INFO [StoreOpener-af3054c22fab85f44f2e3f3ac4020be7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af3054c22fab85f44f2e3f3ac4020be7 columnFamilyName A 2024-11-26T15:27:06,299 DEBUG [StoreOpener-af3054c22fab85f44f2e3f3ac4020be7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:06,299 INFO [StoreOpener-af3054c22fab85f44f2e3f3ac4020be7-1 {}] regionserver.HStore(327): Store=af3054c22fab85f44f2e3f3ac4020be7/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:27:06,299 INFO [StoreOpener-af3054c22fab85f44f2e3f3ac4020be7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:06,301 INFO [StoreOpener-af3054c22fab85f44f2e3f3ac4020be7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af3054c22fab85f44f2e3f3ac4020be7 columnFamilyName B 2024-11-26T15:27:06,301 DEBUG [StoreOpener-af3054c22fab85f44f2e3f3ac4020be7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:06,302 INFO [StoreOpener-af3054c22fab85f44f2e3f3ac4020be7-1 {}] regionserver.HStore(327): Store=af3054c22fab85f44f2e3f3ac4020be7/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:27:06,302 INFO [StoreOpener-af3054c22fab85f44f2e3f3ac4020be7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:06,304 INFO [StoreOpener-af3054c22fab85f44f2e3f3ac4020be7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af3054c22fab85f44f2e3f3ac4020be7 columnFamilyName C 2024-11-26T15:27:06,304 DEBUG [StoreOpener-af3054c22fab85f44f2e3f3ac4020be7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:06,304 INFO [StoreOpener-af3054c22fab85f44f2e3f3ac4020be7-1 {}] regionserver.HStore(327): Store=af3054c22fab85f44f2e3f3ac4020be7/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:27:06,304 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:06,305 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:06,305 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:06,306 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:06,306 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:06,307 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T15:27:06,308 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:06,309 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:27:06,310 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened af3054c22fab85f44f2e3f3ac4020be7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65538292, jitterRate=-0.0234033465385437}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T15:27:06,310 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:06,310 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for af3054c22fab85f44f2e3f3ac4020be7: Running coprocessor pre-open hook at 1732634826294Writing region info on filesystem at 1732634826294Initializing all the Stores at 1732634826295 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634826295Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634826296 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634826296Cleaning up temporary data from old regions at 1732634826306 (+10 ms)Running coprocessor post-open hooks at 1732634826310 (+4 ms)Region opened successfully at 1732634826310 2024-11-26T15:27:06,311 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., pid=120, masterSystemTime=1732634826286 2024-11-26T15:27:06,312 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:06,312 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:06,313 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=af3054c22fab85f44f2e3f3ac4020be7, regionState=OPEN, openSeqNum=2, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:06,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:27:06,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-26T15:27:06,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 in 182 msec 2024-11-26T15:27:06,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-26T15:27:06,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=af3054c22fab85f44f2e3f3ac4020be7, ASSIGN in 340 msec 2024-11-26T15:27:06,318 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T15:27:06,318 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634826318"}]},"ts":"1732634826318"} 2024-11-26T15:27:06,319 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T15:27:06,319 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T15:27:06,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-11-26T15:27:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-26T15:27:07,303 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:07,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@845334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:07,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:07,305 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:07,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:07,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:07,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ef9a73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:07,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:07,306 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:07,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44a02bda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,307 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:07,309 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:07,309 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T15:27:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 6cc2bc8bf733,37081,1732634696319 2024-11-26T15:27:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@60e33871 2024-11-26T15:27:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T15:27:07,310 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T15:27:07,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a6d3d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,312 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:07,312 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:07,312 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@481c652a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:07,314 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:07,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58cb1527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3734923a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,315 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:07,315 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:07,315 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:07,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:07,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ef1f2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:07,316 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e038de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f704f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:07,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:07,318 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:07,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:07,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:07,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60bb6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:07,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:07,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:07,319 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:07,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e100a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b7894c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,321 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:07,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:07,321 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:07,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:07,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:07,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bee3297, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:07,322 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:07,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@624af707, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1447b9d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:07,324 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:07,324 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:07,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:07,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:07,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a08909d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:07,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:07,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:07,325 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:07,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c9417ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f0c5cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:07,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:07,326 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c1d9b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:07,327 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8b30a96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f86346d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,328 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:07,328 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:07,329 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40313d59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:07,330 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:07,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59ced2ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b469189, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,331 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:07,331 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:07,331 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:07,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:07,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:07,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d31dc4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:07,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:07,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:07,332 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:07,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@742fbc8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6240d801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:07,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:07,334 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:07,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:07,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:07,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7673e492, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:07,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:07,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:07,335 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:07,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d871bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e2d9ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:07,336 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:07,336 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cad46fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:07,338 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:07,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58f98eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:07,339 DEBUG [Thread-1970 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:07,339 DEBUG [Thread-1985 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:07,339 DEBUG [Thread-1981 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:07,339 DEBUG [Thread-1979 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:07,340 DEBUG [Thread-1977 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:07,340 DEBUG [Thread-1972 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:07,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:07,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:07,340 DEBUG [Thread-1974 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:07,340 DEBUG [Thread-1968 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:07,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:07,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:07,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:07,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:07,340 DEBUG [Thread-1966 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:07,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:07,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:07,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:07,341 DEBUG [Thread-1983 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:07,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:07,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:27:07,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:07,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:07,342 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:07,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:07,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:07,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-26T15:27:07,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:07,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:07,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:07,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:07,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:07,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:27:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T15:27:07,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:07,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:27:07,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:27:07,346 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:27:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:27:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:27:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:27:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:27:07,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:27:07,348 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:27:07,348 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:27:07,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:27:07,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2] 2024-11-26T15:27:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:07,356 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:07,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T15:27:07,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/e3b922e6ccc2439bb3f5a598f943ecdf is 50, key is test_row_0/A:col10/1732634827351/Put/seqid=0 2024-11-26T15:27:07,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48188 deadline: 1732634887385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:07,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1732634887387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:07,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:07,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48120 deadline: 1732634887394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1732634887394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:07,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634887395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:07,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742262_1438 (size=16681) 2024-11-26T15:27:07,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/e3b922e6ccc2439bb3f5a598f943ecdf 2024-11-26T15:27:07,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/a1c929e9ac4a4258907babdc44fac295 is 50, key is test_row_0/B:col10/1732634827351/Put/seqid=0 2024-11-26T15:27:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T15:27:07,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742263_1439 (size=12001) 2024-11-26T15:27:07,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T15:27:07,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:07,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:07,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:07,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:07,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:07,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48188 deadline: 1732634887503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:07,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1732634887503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:07,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48120 deadline: 1732634887503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:07,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1732634887504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:07,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:07,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634887513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:07,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:07,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T15:27:07,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:07,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:07,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:07,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:07,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T15:27:07,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48188 deadline: 1732634887723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:07,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:07,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1732634887723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:07,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48120 deadline: 1732634887723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:07,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1732634887723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:07,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634887723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:07,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:07,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:07,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T15:27:07,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:07,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:07,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:07,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:07,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:07,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:07,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/a1c929e9ac4a4258907babdc44fac295 2024-11-26T15:27:07,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/041e7f5faa3c4c5f94dacc12d645908a is 50, key is test_row_0/C:col10/1732634827351/Put/seqid=0 2024-11-26T15:27:07,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742264_1440 (size=12001) 2024-11-26T15:27:07,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T15:27:07,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:07,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:07,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:07,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:07,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:07,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T15:27:08,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48188 deadline: 1732634888032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1732634888032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1732634888033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48120 deadline: 1732634888033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634888042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T15:27:08,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:08,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:08,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:08,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:08,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:08,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:08,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T15:27:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:08,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:08,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/041e7f5faa3c4c5f94dacc12d645908a 2024-11-26T15:27:08,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/e3b922e6ccc2439bb3f5a598f943ecdf as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/e3b922e6ccc2439bb3f5a598f943ecdf 2024-11-26T15:27:08,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/e3b922e6ccc2439bb3f5a598f943ecdf, entries=250, sequenceid=14, filesize=16.3 K 2024-11-26T15:27:08,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/a1c929e9ac4a4258907babdc44fac295 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a1c929e9ac4a4258907babdc44fac295 2024-11-26T15:27:08,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a1c929e9ac4a4258907babdc44fac295, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T15:27:08,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/041e7f5faa3c4c5f94dacc12d645908a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/041e7f5faa3c4c5f94dacc12d645908a 2024-11-26T15:27:08,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/041e7f5faa3c4c5f94dacc12d645908a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T15:27:08,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for af3054c22fab85f44f2e3f3ac4020be7 in 938ms, sequenceid=14, compaction requested=false 2024-11-26T15:27:08,294 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T15:27:08,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:08,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T15:27:08,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:08,420 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T15:27:08,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/3cb8afffbf9b464b9de5ddf5e04dc9eb is 50, key is test_row_0/A:col10/1732634827393/Put/seqid=0 2024-11-26T15:27:08,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742265_1441 (size=12001) 2024-11-26T15:27:08,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T15:27:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:08,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:08,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634888552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1732634888553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48120 deadline: 1732634888554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1732634888554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48188 deadline: 1732634888556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1732634888662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48120 deadline: 1732634888673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1732634888673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48188 deadline: 1732634888673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,829 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/3cb8afffbf9b464b9de5ddf5e04dc9eb 2024-11-26T15:27:08,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/0496c00f152a4c27a1a6db885fd7cbe3 is 50, key is test_row_0/B:col10/1732634827393/Put/seqid=0 2024-11-26T15:27:08,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742266_1442 (size=12001) 2024-11-26T15:27:08,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1732634888872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48120 deadline: 1732634888883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1732634888883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48188 deadline: 1732634888883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:09,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1732634889182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:09,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:09,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48120 deadline: 1732634889192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:09,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1732634889192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:09,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:09,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48188 deadline: 1732634889193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:09,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:09,238 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/0496c00f152a4c27a1a6db885fd7cbe3 2024-11-26T15:27:09,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/c4bfe7db581142f699cd4380031e9c19 is 50, key is test_row_0/C:col10/1732634827393/Put/seqid=0 2024-11-26T15:27:09,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742267_1443 (size=12001) 2024-11-26T15:27:09,249 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/c4bfe7db581142f699cd4380031e9c19 2024-11-26T15:27:09,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/3cb8afffbf9b464b9de5ddf5e04dc9eb as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/3cb8afffbf9b464b9de5ddf5e04dc9eb 2024-11-26T15:27:09,255 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/3cb8afffbf9b464b9de5ddf5e04dc9eb, entries=150, sequenceid=38, filesize=11.7 K 2024-11-26T15:27:09,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/0496c00f152a4c27a1a6db885fd7cbe3 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0496c00f152a4c27a1a6db885fd7cbe3 2024-11-26T15:27:09,258 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0496c00f152a4c27a1a6db885fd7cbe3, entries=150, sequenceid=38, filesize=11.7 K 2024-11-26T15:27:09,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/c4bfe7db581142f699cd4380031e9c19 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/c4bfe7db581142f699cd4380031e9c19 2024-11-26T15:27:09,262 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/c4bfe7db581142f699cd4380031e9c19, entries=150, sequenceid=38, filesize=11.7 K 2024-11-26T15:27:09,263 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for af3054c22fab85f44f2e3f3ac4020be7 in 842ms, sequenceid=38, compaction requested=false 2024-11-26T15:27:09,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:09,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:09,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-26T15:27:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-26T15:27:09,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-26T15:27:09,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9160 sec 2024-11-26T15:27:09,267 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.9250 sec 2024-11-26T15:27:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T15:27:09,493 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:09,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:27:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-26T15:27:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T15:27:09,495 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:27:09,495 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:27:09,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:27:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:09,565 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:09,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-26T15:27:09,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/ddc9ce54522d4270bcc6aea7d039115b is 50, key is test_row_0/A:col10/1732634828553/Put/seqid=0 2024-11-26T15:27:09,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742268_1444 (size=14341) 2024-11-26T15:27:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T15:27:09,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T15:27:09,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:09,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:09,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:09,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:09,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:09,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634889691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:09,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1732634889693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:09,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48120 deadline: 1732634889703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1732634889703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:09,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48188 deadline: 1732634889703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:09,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T15:27:09,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:09,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:09,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:09,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:09,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:09,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634889802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T15:27:09,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T15:27:09,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:09,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:09,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:09,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:09,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:09,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:09,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/ddc9ce54522d4270bcc6aea7d039115b 2024-11-26T15:27:09,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/338bf7a3d4ac434184fa4e87c957cc7d is 50, key is test_row_0/B:col10/1732634828553/Put/seqid=0 2024-11-26T15:27:09,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742269_1445 (size=12001) 2024-11-26T15:27:10,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634890013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:10,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T15:27:10,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:10,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:10,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:10,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T15:27:10,220 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T15:27:10,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T15:27:10,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:10,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:10,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:10,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634890322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:10,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/338bf7a3d4ac434184fa4e87c957cc7d 2024-11-26T15:27:10,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/86d93f5f184845a185dad5d9b80f6d08 is 50, key is test_row_0/C:col10/1732634828553/Put/seqid=0 2024-11-26T15:27:10,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742270_1446 (size=12001) 2024-11-26T15:27:10,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T15:27:10,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:10,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:10,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:10,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T15:27:10,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:10,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:10,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:10,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T15:27:10,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T15:27:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:10,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:10,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1732634890712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:10,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:10,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1732634890713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:10,719 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in af3054c22fab85f44f2e3f3ac4020be7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:10,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:10,719 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in af3054c22fab85f44f2e3f3ac4020be7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:10,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:10,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:10,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:10,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:10,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:10,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48120 deadline: 1732634890722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:10,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48188 deadline: 1732634890722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:10,732 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in af3054c22fab85f44f2e3f3ac4020be7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:10,732 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in af3054c22fab85f44f2e3f3ac4020be7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:10,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:10,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:10,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:10,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:10,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:10,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:10,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/86d93f5f184845a185dad5d9b80f6d08 2024-11-26T15:27:10,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/ddc9ce54522d4270bcc6aea7d039115b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ddc9ce54522d4270bcc6aea7d039115b 2024-11-26T15:27:10,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ddc9ce54522d4270bcc6aea7d039115b, entries=200, sequenceid=51, filesize=14.0 K 2024-11-26T15:27:10,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/338bf7a3d4ac434184fa4e87c957cc7d as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/338bf7a3d4ac434184fa4e87c957cc7d 2024-11-26T15:27:10,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/338bf7a3d4ac434184fa4e87c957cc7d, entries=150, sequenceid=51, filesize=11.7 K 2024-11-26T15:27:10,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/86d93f5f184845a185dad5d9b80f6d08 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/86d93f5f184845a185dad5d9b80f6d08 2024-11-26T15:27:10,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/86d93f5f184845a185dad5d9b80f6d08, entries=150, sequenceid=51, filesize=11.7 K 2024-11-26T15:27:10,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for af3054c22fab85f44f2e3f3ac4020be7 in 1271ms, sequenceid=51, compaction requested=true 2024-11-26T15:27:10,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:10,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:27:10,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:10,837 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:10,837 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:10,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:27:10,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:10,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:27:10,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:10,838 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:10,838 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:10,838 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/A is initiating minor compaction (all files) 2024-11-26T15:27:10,838 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/B is initiating minor compaction (all files) 2024-11-26T15:27:10,838 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/A in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:10,838 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/e3b922e6ccc2439bb3f5a598f943ecdf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/3cb8afffbf9b464b9de5ddf5e04dc9eb, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ddc9ce54522d4270bcc6aea7d039115b] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=42.0 K 2024-11-26T15:27:10,840 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/B in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:10,841 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a1c929e9ac4a4258907babdc44fac295, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0496c00f152a4c27a1a6db885fd7cbe3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/338bf7a3d4ac434184fa4e87c957cc7d] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=35.2 K 2024-11-26T15:27:10,843 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3b922e6ccc2439bb3f5a598f943ecdf, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732634827351 2024-11-26T15:27:10,843 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting a1c929e9ac4a4258907babdc44fac295, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732634827351 2024-11-26T15:27:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:10,843 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 0496c00f152a4c27a1a6db885fd7cbe3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732634827392 2024-11-26T15:27:10,843 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cb8afffbf9b464b9de5ddf5e04dc9eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732634827392 2024-11-26T15:27:10,844 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 338bf7a3d4ac434184fa4e87c957cc7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732634828553 2024-11-26T15:27:10,844 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddc9ce54522d4270bcc6aea7d039115b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732634828546 2024-11-26T15:27:10,845 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:10,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T15:27:10,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/16d2c6491fee40989d02af38f4a9539f is 50, key is test_row_0/A:col10/1732634830841/Put/seqid=0 2024-11-26T15:27:10,858 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#A#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:10,858 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/6691ea8ef777474dbe517460cc90dd2e is 50, key is test_row_0/A:col10/1732634828553/Put/seqid=0 2024-11-26T15:27:10,861 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#B#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:10,862 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/7e0e2727cc0046659f7a636995b8ec2f is 50, key is test_row_0/B:col10/1732634828553/Put/seqid=0 2024-11-26T15:27:10,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T15:27:10,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:10,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:10,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:10,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:10,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742271_1447 (size=14341) 2024-11-26T15:27:10,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/16d2c6491fee40989d02af38f4a9539f 2024-11-26T15:27:10,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742272_1448 (size=12104) 2024-11-26T15:27:10,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742273_1449 (size=12104) 2024-11-26T15:27:10,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/63743d7ca05840a890c2fd122f4bed7f is 50, key is test_row_0/B:col10/1732634830841/Put/seqid=0 2024-11-26T15:27:10,908 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/6691ea8ef777474dbe517460cc90dd2e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/6691ea8ef777474dbe517460cc90dd2e 2024-11-26T15:27:10,908 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/7e0e2727cc0046659f7a636995b8ec2f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/7e0e2727cc0046659f7a636995b8ec2f 2024-11-26T15:27:10,912 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/A of af3054c22fab85f44f2e3f3ac4020be7 into 6691ea8ef777474dbe517460cc90dd2e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:10,912 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:10,912 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/A, priority=13, startTime=1732634830837; duration=0sec 2024-11-26T15:27:10,912 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:10,912 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:A 2024-11-26T15:27:10,913 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:10,913 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/B of af3054c22fab85f44f2e3f3ac4020be7 into 7e0e2727cc0046659f7a636995b8ec2f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:10,913 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:10,914 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/B, priority=13, startTime=1732634830837; duration=0sec 2024-11-26T15:27:10,914 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:10,914 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:B 2024-11-26T15:27:10,915 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:10,915 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/C is initiating minor compaction (all files) 2024-11-26T15:27:10,915 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/C in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:10,915 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/041e7f5faa3c4c5f94dacc12d645908a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/c4bfe7db581142f699cd4380031e9c19, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/86d93f5f184845a185dad5d9b80f6d08] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=35.2 K 2024-11-26T15:27:10,916 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 041e7f5faa3c4c5f94dacc12d645908a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732634827351 2024-11-26T15:27:10,916 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4bfe7db581142f699cd4380031e9c19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732634827392 2024-11-26T15:27:10,917 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86d93f5f184845a185dad5d9b80f6d08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732634828553 2024-11-26T15:27:10,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742274_1450 (size=12001) 2024-11-26T15:27:10,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/63743d7ca05840a890c2fd122f4bed7f 2024-11-26T15:27:10,926 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#C#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:10,926 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/671e6aa921a64b1e9d3039ad6be739a2 is 50, key is test_row_0/C:col10/1732634828553/Put/seqid=0 2024-11-26T15:27:10,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/42b3d987cbea4686a9a056aeb327f1d1 is 50, key is test_row_0/C:col10/1732634830841/Put/seqid=0 2024-11-26T15:27:10,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742275_1451 (size=12104) 2024-11-26T15:27:10,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742276_1452 (size=12001) 2024-11-26T15:27:10,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/42b3d987cbea4686a9a056aeb327f1d1 2024-11-26T15:27:10,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/16d2c6491fee40989d02af38f4a9539f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/16d2c6491fee40989d02af38f4a9539f 2024-11-26T15:27:10,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/16d2c6491fee40989d02af38f4a9539f, entries=200, sequenceid=75, filesize=14.0 K 2024-11-26T15:27:10,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/63743d7ca05840a890c2fd122f4bed7f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/63743d7ca05840a890c2fd122f4bed7f 2024-11-26T15:27:10,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634890943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:10,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/63743d7ca05840a890c2fd122f4bed7f, entries=150, sequenceid=75, filesize=11.7 K 2024-11-26T15:27:10,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/42b3d987cbea4686a9a056aeb327f1d1 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/42b3d987cbea4686a9a056aeb327f1d1 2024-11-26T15:27:10,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/42b3d987cbea4686a9a056aeb327f1d1, entries=150, sequenceid=75, filesize=11.7 K 2024-11-26T15:27:10,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for af3054c22fab85f44f2e3f3ac4020be7 in 107ms, sequenceid=75, compaction requested=false 2024-11-26T15:27:10,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:11,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T15:27:11,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:11,022 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-26T15:27:11,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/7f06f48cc22249bc88dd997907309403 is 50, key is test_row_0/A:col10/1732634830925/Put/seqid=0 2024-11-26T15:27:11,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742277_1453 (size=12001) 2024-11-26T15:27:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:11,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:11,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634891243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:11,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:11,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:11,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:11,338 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/671e6aa921a64b1e9d3039ad6be739a2 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/671e6aa921a64b1e9d3039ad6be739a2 2024-11-26T15:27:11,343 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/C of af3054c22fab85f44f2e3f3ac4020be7 into 671e6aa921a64b1e9d3039ad6be739a2(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:11,343 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:11,343 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/C, priority=13, startTime=1732634830837; duration=0sec 2024-11-26T15:27:11,343 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:11,343 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:C 2024-11-26T15:27:11,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634891363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:11,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:11,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:11,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:11,430 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/7f06f48cc22249bc88dd997907309403 2024-11-26T15:27:11,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/a4a8c5924d4b4200832a245c5ed4260f is 50, key is test_row_0/B:col10/1732634830925/Put/seqid=0 2024-11-26T15:27:11,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742278_1454 (size=12001) 2024-11-26T15:27:11,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634891582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:11,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:11,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:11,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:11,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T15:27:11,849 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/a4a8c5924d4b4200832a245c5ed4260f 2024-11-26T15:27:11,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/c75a79c25ccd4dbb8c54546fefb66f32 is 50, key is test_row_0/C:col10/1732634830925/Put/seqid=0 2024-11-26T15:27:11,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742279_1455 (size=12001) 2024-11-26T15:27:11,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634891893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:11,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:11,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:11,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:12,265 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/c75a79c25ccd4dbb8c54546fefb66f32 2024-11-26T15:27:12,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/7f06f48cc22249bc88dd997907309403 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/7f06f48cc22249bc88dd997907309403 2024-11-26T15:27:12,271 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/7f06f48cc22249bc88dd997907309403, entries=150, sequenceid=89, filesize=11.7 K 2024-11-26T15:27:12,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/a4a8c5924d4b4200832a245c5ed4260f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a4a8c5924d4b4200832a245c5ed4260f 2024-11-26T15:27:12,274 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a4a8c5924d4b4200832a245c5ed4260f, entries=150, sequenceid=89, filesize=11.7 K 2024-11-26T15:27:12,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/c75a79c25ccd4dbb8c54546fefb66f32 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/c75a79c25ccd4dbb8c54546fefb66f32 2024-11-26T15:27:12,278 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/c75a79c25ccd4dbb8c54546fefb66f32, entries=150, sequenceid=89, filesize=11.7 K 2024-11-26T15:27:12,279 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for af3054c22fab85f44f2e3f3ac4020be7 in 1257ms, sequenceid=89, compaction requested=true 2024-11-26T15:27:12,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:12,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:12,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-26T15:27:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-26T15:27:12,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-26T15:27:12,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7850 sec 2024-11-26T15:27:12,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.7880 sec 2024-11-26T15:27:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:12,414 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:12,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-26T15:27:12,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/92e52de0b319462fb825c3b3f2948f93 is 50, key is test_row_0/A:col10/1732634831197/Put/seqid=0 2024-11-26T15:27:12,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742280_1456 (size=14341) 2024-11-26T15:27:12,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634892454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:12,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634892562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:12,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:12,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:12,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:12,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1732634892733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:12,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1732634892733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:12,738 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in af3054c22fab85f44f2e3f3ac4020be7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:12,738 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in af3054c22fab85f44f2e3f3ac4020be7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:12,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:12,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48120 deadline: 1732634892752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:12,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48188 deadline: 1732634892752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:12,757 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in af3054c22fab85f44f2e3f3ac4020be7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:12,757 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in af3054c22fab85f44f2e3f3ac4020be7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:12,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634892782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:12,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/92e52de0b319462fb825c3b3f2948f93 2024-11-26T15:27:12,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/baaece080dc9445f9354013d0da6ed7a is 50, key is test_row_0/B:col10/1732634831197/Put/seqid=0 2024-11-26T15:27:12,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742281_1457 (size=12001) 2024-11-26T15:27:13,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:13,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634893093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:13,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/baaece080dc9445f9354013d0da6ed7a 2024-11-26T15:27:13,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/2e1f80623b2f4a7c94088b58746ec8b5 is 50, key is test_row_0/C:col10/1732634831197/Put/seqid=0 2024-11-26T15:27:13,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742282_1458 (size=12001) 2024-11-26T15:27:13,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/2e1f80623b2f4a7c94088b58746ec8b5 2024-11-26T15:27:13,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/92e52de0b319462fb825c3b3f2948f93 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/92e52de0b319462fb825c3b3f2948f93 2024-11-26T15:27:13,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/92e52de0b319462fb825c3b3f2948f93, entries=200, sequenceid=115, filesize=14.0 K 2024-11-26T15:27:13,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/baaece080dc9445f9354013d0da6ed7a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/baaece080dc9445f9354013d0da6ed7a 2024-11-26T15:27:13,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/baaece080dc9445f9354013d0da6ed7a, entries=150, sequenceid=115, filesize=11.7 K 2024-11-26T15:27:13,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/2e1f80623b2f4a7c94088b58746ec8b5 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/2e1f80623b2f4a7c94088b58746ec8b5 2024-11-26T15:27:13,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/2e1f80623b2f4a7c94088b58746ec8b5, entries=150, sequenceid=115, filesize=11.7 K 2024-11-26T15:27:13,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for af3054c22fab85f44f2e3f3ac4020be7 in 842ms, sequenceid=115, compaction requested=true 2024-11-26T15:27:13,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:13,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:27:13,256 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:27:13,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:13,256 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:27:13,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:27:13,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:13,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:27:13,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:13,257 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:27:13,257 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:27:13,257 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/A is initiating minor compaction (all files) 2024-11-26T15:27:13,257 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/B is initiating minor compaction (all files) 2024-11-26T15:27:13,257 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/A in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:13,257 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/6691ea8ef777474dbe517460cc90dd2e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/16d2c6491fee40989d02af38f4a9539f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/7f06f48cc22249bc88dd997907309403, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/92e52de0b319462fb825c3b3f2948f93] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=51.5 K 2024-11-26T15:27:13,258 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6691ea8ef777474dbe517460cc90dd2e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732634828553 2024-11-26T15:27:13,258 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/B in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:13,258 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/7e0e2727cc0046659f7a636995b8ec2f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/63743d7ca05840a890c2fd122f4bed7f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a4a8c5924d4b4200832a245c5ed4260f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/baaece080dc9445f9354013d0da6ed7a] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=47.0 K 2024-11-26T15:27:13,260 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e0e2727cc0046659f7a636995b8ec2f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732634828553 2024-11-26T15:27:13,260 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16d2c6491fee40989d02af38f4a9539f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732634829665 2024-11-26T15:27:13,260 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f06f48cc22249bc88dd997907309403, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732634830913 2024-11-26T15:27:13,260 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 63743d7ca05840a890c2fd122f4bed7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732634829665 2024-11-26T15:27:13,261 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92e52de0b319462fb825c3b3f2948f93, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732634831191 2024-11-26T15:27:13,261 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting a4a8c5924d4b4200832a245c5ed4260f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732634830913 2024-11-26T15:27:13,261 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting baaece080dc9445f9354013d0da6ed7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732634831197 2024-11-26T15:27:13,267 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#A#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:13,268 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/96807eae4599410f9a73a691dab165b6 is 50, key is test_row_0/A:col10/1732634831197/Put/seqid=0 2024-11-26T15:27:13,272 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#B#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:13,272 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/721a1958ba12466f9d85e39e87ec948f is 50, key is test_row_0/B:col10/1732634831197/Put/seqid=0 2024-11-26T15:27:13,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742283_1459 (size=12241) 2024-11-26T15:27:13,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742284_1460 (size=12241) 2024-11-26T15:27:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:13,608 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:13,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-26T15:27:13,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/afeeb0eafa434beb8fde2551bd290d2e is 50, key is test_row_0/A:col10/1732634832436/Put/seqid=0 2024-11-26T15:27:13,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742285_1461 (size=14341) 2024-11-26T15:27:13,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/afeeb0eafa434beb8fde2551bd290d2e 2024-11-26T15:27:13,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/8ab9194cdef94a259e05b5099aa3169e is 50, key is test_row_0/B:col10/1732634832436/Put/seqid=0 2024-11-26T15:27:13,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742286_1462 (size=12001) 2024-11-26T15:27:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T15:27:13,652 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:13,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:27:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-26T15:27:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T15:27:13,656 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:27:13,656 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:27:13,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:27:13,688 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/721a1958ba12466f9d85e39e87ec948f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/721a1958ba12466f9d85e39e87ec948f 2024-11-26T15:27:13,697 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/B of af3054c22fab85f44f2e3f3ac4020be7 into 721a1958ba12466f9d85e39e87ec948f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:13,697 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:13,697 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/B, priority=12, startTime=1732634833256; duration=0sec 2024-11-26T15:27:13,697 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:13,697 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:B 2024-11-26T15:27:13,697 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:27:13,698 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:27:13,698 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/C is initiating minor compaction (all files) 2024-11-26T15:27:13,698 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/C in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:13,698 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/671e6aa921a64b1e9d3039ad6be739a2, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/42b3d987cbea4686a9a056aeb327f1d1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/c75a79c25ccd4dbb8c54546fefb66f32, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/2e1f80623b2f4a7c94088b58746ec8b5] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=47.0 K 2024-11-26T15:27:13,699 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 671e6aa921a64b1e9d3039ad6be739a2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732634828553 2024-11-26T15:27:13,699 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/96807eae4599410f9a73a691dab165b6 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/96807eae4599410f9a73a691dab165b6 2024-11-26T15:27:13,699 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 42b3d987cbea4686a9a056aeb327f1d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732634829665 2024-11-26T15:27:13,699 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting c75a79c25ccd4dbb8c54546fefb66f32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732634830913 2024-11-26T15:27:13,699 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e1f80623b2f4a7c94088b58746ec8b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732634831197 2024-11-26T15:27:13,702 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/A of af3054c22fab85f44f2e3f3ac4020be7 into 96807eae4599410f9a73a691dab165b6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:13,703 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:13,703 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/A, priority=12, startTime=1732634833256; duration=0sec 2024-11-26T15:27:13,703 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:13,703 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:A 2024-11-26T15:27:13,710 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#C#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:13,710 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/2550df2870df4125a92832e20b295bc5 is 50, key is test_row_0/C:col10/1732634831197/Put/seqid=0 2024-11-26T15:27:13,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742287_1463 (size=12241) 2024-11-26T15:27:13,718 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/2550df2870df4125a92832e20b295bc5 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/2550df2870df4125a92832e20b295bc5 2024-11-26T15:27:13,723 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/C of af3054c22fab85f44f2e3f3ac4020be7 into 2550df2870df4125a92832e20b295bc5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:13,723 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:13,723 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/C, priority=12, startTime=1732634833256; duration=0sec 2024-11-26T15:27:13,723 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:13,724 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:C 2024-11-26T15:27:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T15:27:13,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:13,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634893794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:13,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T15:27:13,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:13,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:13,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:13,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:13,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:13,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:13,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634893902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:13,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:13,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:13,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:13,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T15:27:13,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:13,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:13,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:13,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:13,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T15:27:14,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/8ab9194cdef94a259e05b5099aa3169e 2024-11-26T15:27:14,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/de076ff6ed9d480a92ff8d40957be1ad is 50, key is test_row_0/C:col10/1732634832436/Put/seqid=0 2024-11-26T15:27:14,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742288_1464 (size=12001) 2024-11-26T15:27:14,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T15:27:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:14,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:14,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:14,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634894112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:14,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T15:27:14,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:14,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:14,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:14,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:14,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T15:27:14,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T15:27:14,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:14,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:14,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:14,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:14,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:14,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634894422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:14,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:14,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:14,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:14,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/de076ff6ed9d480a92ff8d40957be1ad 2024-11-26T15:27:14,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/afeeb0eafa434beb8fde2551bd290d2e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/afeeb0eafa434beb8fde2551bd290d2e 2024-11-26T15:27:14,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/afeeb0eafa434beb8fde2551bd290d2e, entries=200, sequenceid=127, filesize=14.0 K 2024-11-26T15:27:14,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/8ab9194cdef94a259e05b5099aa3169e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/8ab9194cdef94a259e05b5099aa3169e 2024-11-26T15:27:14,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/8ab9194cdef94a259e05b5099aa3169e, entries=150, sequenceid=127, filesize=11.7 K 2024-11-26T15:27:14,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/de076ff6ed9d480a92ff8d40957be1ad as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/de076ff6ed9d480a92ff8d40957be1ad 2024-11-26T15:27:14,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/de076ff6ed9d480a92ff8d40957be1ad, entries=150, sequenceid=127, filesize=11.7 K 2024-11-26T15:27:14,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for af3054c22fab85f44f2e3f3ac4020be7 in 864ms, sequenceid=127, compaction requested=false 2024-11-26T15:27:14,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:14,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T15:27:14,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:14,572 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T15:27:14,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/f2b43c1e31254c20a82211c68c363f55 is 50, key is test_row_0/A:col10/1732634833751/Put/seqid=0 2024-11-26T15:27:14,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742289_1465 (size=12151) 2024-11-26T15:27:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T15:27:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:14,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:14,981 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/f2b43c1e31254c20a82211c68c363f55 2024-11-26T15:27:14,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634894981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:14,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/c48cf3242d57440dbc14d2b3c0647f03 is 50, key is test_row_0/B:col10/1732634833751/Put/seqid=0 2024-11-26T15:27:14,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742290_1466 (size=12151) 2024-11-26T15:27:15,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634895092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:15,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634895302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:15,390 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/c48cf3242d57440dbc14d2b3c0647f03 2024-11-26T15:27:15,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/4c244f684e1b4ca2af79a803eea61a94 is 50, key is test_row_0/C:col10/1732634833751/Put/seqid=0 2024-11-26T15:27:15,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742291_1467 (size=12151) 2024-11-26T15:27:15,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634895622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:15,800 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/4c244f684e1b4ca2af79a803eea61a94 2024-11-26T15:27:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T15:27:15,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/f2b43c1e31254c20a82211c68c363f55 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/f2b43c1e31254c20a82211c68c363f55 2024-11-26T15:27:15,806 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/f2b43c1e31254c20a82211c68c363f55, entries=150, sequenceid=154, filesize=11.9 K 2024-11-26T15:27:15,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/c48cf3242d57440dbc14d2b3c0647f03 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/c48cf3242d57440dbc14d2b3c0647f03 2024-11-26T15:27:15,810 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/c48cf3242d57440dbc14d2b3c0647f03, entries=150, sequenceid=154, filesize=11.9 K 2024-11-26T15:27:15,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/4c244f684e1b4ca2af79a803eea61a94 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/4c244f684e1b4ca2af79a803eea61a94 2024-11-26T15:27:15,813 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/4c244f684e1b4ca2af79a803eea61a94, entries=150, sequenceid=154, filesize=11.9 K 2024-11-26T15:27:15,814 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for af3054c22fab85f44f2e3f3ac4020be7 in 1242ms, sequenceid=154, compaction requested=true 2024-11-26T15:27:15,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:15,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:15,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-26T15:27:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-26T15:27:15,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-26T15:27:15,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1590 sec 2024-11-26T15:27:15,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.1640 sec 2024-11-26T15:27:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:16,136 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:16,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-26T15:27:16,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/b39d5e9388394e3f9eea390354d7aae4 is 50, key is test_row_0/A:col10/1732634836135/Put/seqid=0 2024-11-26T15:27:16,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742292_1468 (size=14541) 2024-11-26T15:27:16,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/b39d5e9388394e3f9eea390354d7aae4 2024-11-26T15:27:16,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/42077007442143e4808c81a9bbfef024 is 50, key is test_row_0/B:col10/1732634836135/Put/seqid=0 2024-11-26T15:27:16,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742293_1469 (size=12151) 2024-11-26T15:27:16,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634896262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:16,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634896372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:16,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:16,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:16,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:16,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/42077007442143e4808c81a9bbfef024 2024-11-26T15:27:16,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/fc29c0a4cd174319bae4b71c0cfcf4b1 is 50, key is test_row_0/C:col10/1732634836135/Put/seqid=0 2024-11-26T15:27:16,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742294_1470 (size=12151) 2024-11-26T15:27:16,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634896583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:16,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:16,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1732634896762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:16,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:16,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1732634896762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:16,768 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in af3054c22fab85f44f2e3f3ac4020be7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:16,768 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in af3054c22fab85f44f2e3f3ac4020be7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:16,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48120 deadline: 1732634896782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:16,788 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in af3054c22fab85f44f2e3f3ac4020be7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:16,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48188 deadline: 1732634896793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:16,797 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in af3054c22fab85f44f2e3f3ac4020be7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:16,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634896903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:16,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/fc29c0a4cd174319bae4b71c0cfcf4b1 2024-11-26T15:27:16,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/b39d5e9388394e3f9eea390354d7aae4 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/b39d5e9388394e3f9eea390354d7aae4 2024-11-26T15:27:16,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/b39d5e9388394e3f9eea390354d7aae4, entries=200, sequenceid=167, filesize=14.2 K 2024-11-26T15:27:16,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/42077007442143e4808c81a9bbfef024 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/42077007442143e4808c81a9bbfef024 2024-11-26T15:27:16,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/42077007442143e4808c81a9bbfef024, entries=150, sequenceid=167, filesize=11.9 K 2024-11-26T15:27:16,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/fc29c0a4cd174319bae4b71c0cfcf4b1 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/fc29c0a4cd174319bae4b71c0cfcf4b1 2024-11-26T15:27:16,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/fc29c0a4cd174319bae4b71c0cfcf4b1, entries=150, sequenceid=167, filesize=11.9 K 2024-11-26T15:27:16,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for af3054c22fab85f44f2e3f3ac4020be7 in 840ms, sequenceid=167, compaction requested=true 2024-11-26T15:27:16,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:16,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:27:16,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:16,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:27:16,976 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:27:16,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:16,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:27:16,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:16,977 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:27:16,977 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53274 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:27:16,977 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:27:16,977 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/A is initiating minor compaction (all files) 2024-11-26T15:27:16,977 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/B is initiating minor compaction (all files) 2024-11-26T15:27:16,977 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/A in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:16,977 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/B in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:16,977 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/96807eae4599410f9a73a691dab165b6, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/afeeb0eafa434beb8fde2551bd290d2e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/f2b43c1e31254c20a82211c68c363f55, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/b39d5e9388394e3f9eea390354d7aae4] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=52.0 K 2024-11-26T15:27:16,977 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/721a1958ba12466f9d85e39e87ec948f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/8ab9194cdef94a259e05b5099aa3169e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/c48cf3242d57440dbc14d2b3c0647f03, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/42077007442143e4808c81a9bbfef024] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=47.4 K 2024-11-26T15:27:16,978 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 721a1958ba12466f9d85e39e87ec948f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732634831197 2024-11-26T15:27:16,979 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96807eae4599410f9a73a691dab165b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732634831197 2024-11-26T15:27:16,979 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ab9194cdef94a259e05b5099aa3169e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732634832436 2024-11-26T15:27:16,979 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting afeeb0eafa434beb8fde2551bd290d2e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732634832436 2024-11-26T15:27:16,979 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting c48cf3242d57440dbc14d2b3c0647f03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732634833751 2024-11-26T15:27:16,979 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2b43c1e31254c20a82211c68c363f55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732634833751 2024-11-26T15:27:16,979 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 42077007442143e4808c81a9bbfef024, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732634834956 2024-11-26T15:27:16,979 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting b39d5e9388394e3f9eea390354d7aae4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732634834956 2024-11-26T15:27:16,985 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#B#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:16,985 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#A#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:16,985 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/201e8b53597d4a52ab475325aced99e7 is 50, key is test_row_0/B:col10/1732634836135/Put/seqid=0 2024-11-26T15:27:16,985 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/cf0f2e2c3cbd4ca5875ad19016ffe1c4 is 50, key is test_row_0/A:col10/1732634836135/Put/seqid=0 2024-11-26T15:27:16,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742295_1471 (size=12527) 2024-11-26T15:27:16,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742296_1472 (size=12527) 2024-11-26T15:27:17,400 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/cf0f2e2c3cbd4ca5875ad19016ffe1c4 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/cf0f2e2c3cbd4ca5875ad19016ffe1c4 2024-11-26T15:27:17,400 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/201e8b53597d4a52ab475325aced99e7 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/201e8b53597d4a52ab475325aced99e7 2024-11-26T15:27:17,403 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/A of af3054c22fab85f44f2e3f3ac4020be7 into cf0f2e2c3cbd4ca5875ad19016ffe1c4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:17,403 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/B of af3054c22fab85f44f2e3f3ac4020be7 into 201e8b53597d4a52ab475325aced99e7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:17,403 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:17,403 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:17,403 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/A, priority=12, startTime=1732634836976; duration=0sec 2024-11-26T15:27:17,403 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/B, priority=12, startTime=1732634836976; duration=0sec 2024-11-26T15:27:17,404 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:17,404 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:A 2024-11-26T15:27:17,404 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:17,404 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:B 2024-11-26T15:27:17,404 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:27:17,404 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:27:17,404 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/C is initiating minor compaction (all files) 2024-11-26T15:27:17,404 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/C in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:17,404 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/2550df2870df4125a92832e20b295bc5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/de076ff6ed9d480a92ff8d40957be1ad, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/4c244f684e1b4ca2af79a803eea61a94, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/fc29c0a4cd174319bae4b71c0cfcf4b1] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=47.4 K 2024-11-26T15:27:17,405 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2550df2870df4125a92832e20b295bc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732634831197 2024-11-26T15:27:17,405 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting de076ff6ed9d480a92ff8d40957be1ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732634832436 2024-11-26T15:27:17,405 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c244f684e1b4ca2af79a803eea61a94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732634833751 2024-11-26T15:27:17,405 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc29c0a4cd174319bae4b71c0cfcf4b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732634834956 2024-11-26T15:27:17,412 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#C#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:17,412 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/15aa8e57995a4a4aa32e8ef1c14f98b5 is 50, key is test_row_0/C:col10/1732634836135/Put/seqid=0 2024-11-26T15:27:17,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742297_1473 (size=12527) 2024-11-26T15:27:17,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:17,426 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:17,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T15:27:17,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/625cd03d12b7475b86a97bb61cff7089 is 50, key is test_row_0/A:col10/1732634836252/Put/seqid=0 2024-11-26T15:27:17,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742298_1474 (size=14541) 2024-11-26T15:27:17,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:17,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634897471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:17,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:17,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634897582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:17,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634897792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T15:27:17,813 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:17,813 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:27:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-26T15:27:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T15:27:17,814 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:27:17,815 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:27:17,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:27:17,822 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/15aa8e57995a4a4aa32e8ef1c14f98b5 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/15aa8e57995a4a4aa32e8ef1c14f98b5 2024-11-26T15:27:17,826 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/C of af3054c22fab85f44f2e3f3ac4020be7 into 15aa8e57995a4a4aa32e8ef1c14f98b5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:17,826 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:17,826 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/C, priority=12, startTime=1732634836976; duration=0sec 2024-11-26T15:27:17,826 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:17,826 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:C 2024-11-26T15:27:17,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/625cd03d12b7475b86a97bb61cff7089 2024-11-26T15:27:17,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/a8b91bb4dfb047d68d59b5af58c7e581 is 50, key is test_row_0/B:col10/1732634836252/Put/seqid=0 2024-11-26T15:27:17,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742299_1475 (size=12151) 2024-11-26T15:27:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T15:27:17,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T15:27:17,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:17,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:17,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:17,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:17,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:18,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634898103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:18,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T15:27:18,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:18,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T15:27:18,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/a8b91bb4dfb047d68d59b5af58c7e581 2024-11-26T15:27:18,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/1b1b106ee4714d8da77c3e08a89173b2 is 50, key is test_row_0/C:col10/1732634836252/Put/seqid=0 2024-11-26T15:27:18,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742300_1476 (size=12151) 2024-11-26T15:27:18,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T15:27:18,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:18,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:18,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:18,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:18,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:18,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T15:27:18,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:18,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:18,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:18,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:18,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T15:27:18,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T15:27:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:18,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:18,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634898622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:18,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:18,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:18,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:18,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/1b1b106ee4714d8da77c3e08a89173b2 2024-11-26T15:27:18,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/625cd03d12b7475b86a97bb61cff7089 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/625cd03d12b7475b86a97bb61cff7089 2024-11-26T15:27:18,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/625cd03d12b7475b86a97bb61cff7089, entries=200, sequenceid=193, filesize=14.2 K 2024-11-26T15:27:18,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/a8b91bb4dfb047d68d59b5af58c7e581 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a8b91bb4dfb047d68d59b5af58c7e581 2024-11-26T15:27:18,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a8b91bb4dfb047d68d59b5af58c7e581, entries=150, sequenceid=193, filesize=11.9 K 2024-11-26T15:27:18,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/1b1b106ee4714d8da77c3e08a89173b2 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/1b1b106ee4714d8da77c3e08a89173b2 2024-11-26T15:27:18,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/1b1b106ee4714d8da77c3e08a89173b2, entries=150, sequenceid=193, filesize=11.9 K 2024-11-26T15:27:18,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for af3054c22fab85f44f2e3f3ac4020be7 in 1250ms, sequenceid=193, compaction requested=false 2024-11-26T15:27:18,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:18,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-26T15:27:18,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:18,730 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-26T15:27:18,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/304b88f8b5c24c75a71b35cfcfb50b00 is 50, key is test_row_0/A:col10/1732634837463/Put/seqid=0 2024-11-26T15:27:18,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742301_1477 (size=12151) 2024-11-26T15:27:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T15:27:19,137 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/304b88f8b5c24c75a71b35cfcfb50b00 2024-11-26T15:27:19,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/5d91d10021bf45eabcab4af357eb7911 is 50, key is test_row_0/B:col10/1732634837463/Put/seqid=0 2024-11-26T15:27:19,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742302_1478 (size=12151) 2024-11-26T15:27:19,146 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/5d91d10021bf45eabcab4af357eb7911 2024-11-26T15:27:19,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/47b06f17a01c4194a3bbed6e4de669e3 is 50, key is test_row_0/C:col10/1732634837463/Put/seqid=0 2024-11-26T15:27:19,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742303_1479 (size=12151) 2024-11-26T15:27:19,553 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/47b06f17a01c4194a3bbed6e4de669e3 2024-11-26T15:27:19,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/304b88f8b5c24c75a71b35cfcfb50b00 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/304b88f8b5c24c75a71b35cfcfb50b00 2024-11-26T15:27:19,559 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/304b88f8b5c24c75a71b35cfcfb50b00, entries=150, sequenceid=206, filesize=11.9 K 2024-11-26T15:27:19,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/5d91d10021bf45eabcab4af357eb7911 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/5d91d10021bf45eabcab4af357eb7911 2024-11-26T15:27:19,563 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/5d91d10021bf45eabcab4af357eb7911, entries=150, sequenceid=206, filesize=11.9 K 2024-11-26T15:27:19,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/47b06f17a01c4194a3bbed6e4de669e3 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/47b06f17a01c4194a3bbed6e4de669e3 2024-11-26T15:27:19,566 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/47b06f17a01c4194a3bbed6e4de669e3, entries=150, sequenceid=206, filesize=11.9 K 2024-11-26T15:27:19,567 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for af3054c22fab85f44f2e3f3ac4020be7 in 837ms, sequenceid=206, compaction requested=true 2024-11-26T15:27:19,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:19,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:19,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-26T15:27:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-26T15:27:19,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-26T15:27:19,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7530 sec 2024-11-26T15:27:19,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7570 sec 2024-11-26T15:27:19,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:19,662 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:19,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T15:27:19,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/ea7e699ada844224900243d7ce9213e3 is 50, key is test_row_0/A:col10/1732634839661/Put/seqid=0 2024-11-26T15:27:19,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742304_1480 (size=16931) 2024-11-26T15:27:19,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634899792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:19,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634899913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:19,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:19,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:19,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T15:27:19,962 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:19,963 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:27:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-26T15:27:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T15:27:19,964 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:27:19,965 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:27:19,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:27:20,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/ea7e699ada844224900243d7ce9213e3 2024-11-26T15:27:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T15:27:20,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/93bcd23a44aa403ba6d02051a98cccbf is 50, key is test_row_0/B:col10/1732634839661/Put/seqid=0 2024-11-26T15:27:20,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742305_1481 (size=12151) 2024-11-26T15:27:20,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T15:27:20,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:20,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:20,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:20,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:20,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634900122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:20,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:20,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:20,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:20,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T15:27:20,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:20,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:20,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:20,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T15:27:20,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T15:27:20,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:20,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:20,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:20,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634900433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:20,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/93bcd23a44aa403ba6d02051a98cccbf 2024-11-26T15:27:20,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/ff3789757616488d80ac3435ec88a623 is 50, key is test_row_0/C:col10/1732634839661/Put/seqid=0 2024-11-26T15:27:20,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742306_1482 (size=12151) 2024-11-26T15:27:20,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T15:27:20,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:20,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:20,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:20,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T15:27:20,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T15:27:20,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:20,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:20,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:20,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T15:27:20,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:20,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:20,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:20,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:20,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/ff3789757616488d80ac3435ec88a623 2024-11-26T15:27:20,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/ea7e699ada844224900243d7ce9213e3 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ea7e699ada844224900243d7ce9213e3 2024-11-26T15:27:20,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ea7e699ada844224900243d7ce9213e3, entries=250, sequenceid=217, filesize=16.5 K 2024-11-26T15:27:20,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/93bcd23a44aa403ba6d02051a98cccbf as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/93bcd23a44aa403ba6d02051a98cccbf 2024-11-26T15:27:20,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/93bcd23a44aa403ba6d02051a98cccbf, entries=150, sequenceid=217, filesize=11.9 K 2024-11-26T15:27:20,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/ff3789757616488d80ac3435ec88a623 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/ff3789757616488d80ac3435ec88a623 2024-11-26T15:27:20,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/ff3789757616488d80ac3435ec88a623, entries=150, sequenceid=217, filesize=11.9 K 2024-11-26T15:27:20,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for af3054c22fab85f44f2e3f3ac4020be7 in 1236ms, sequenceid=217, compaction requested=true 2024-11-26T15:27:20,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:20,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:27:20,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:20,898 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:27:20,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:27:20,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:20,898 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:27:20,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:27:20,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:20,902 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:27:20,902 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56150 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:27:20,902 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/A is initiating minor compaction (all files) 2024-11-26T15:27:20,902 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/B is initiating minor compaction (all files) 2024-11-26T15:27:20,902 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/B in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:20,902 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/A in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:20,903 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/cf0f2e2c3cbd4ca5875ad19016ffe1c4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/625cd03d12b7475b86a97bb61cff7089, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/304b88f8b5c24c75a71b35cfcfb50b00, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ea7e699ada844224900243d7ce9213e3] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=54.8 K 2024-11-26T15:27:20,903 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/201e8b53597d4a52ab475325aced99e7, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a8b91bb4dfb047d68d59b5af58c7e581, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/5d91d10021bf45eabcab4af357eb7911, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/93bcd23a44aa403ba6d02051a98cccbf] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=47.8 K 2024-11-26T15:27:20,903 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 201e8b53597d4a52ab475325aced99e7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732634834956 2024-11-26T15:27:20,903 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf0f2e2c3cbd4ca5875ad19016ffe1c4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732634834956 2024-11-26T15:27:20,903 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting a8b91bb4dfb047d68d59b5af58c7e581, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732634836252 2024-11-26T15:27:20,903 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 625cd03d12b7475b86a97bb61cff7089, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732634836248 2024-11-26T15:27:20,903 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d91d10021bf45eabcab4af357eb7911, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732634837433 2024-11-26T15:27:20,904 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 304b88f8b5c24c75a71b35cfcfb50b00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732634837433 2024-11-26T15:27:20,904 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea7e699ada844224900243d7ce9213e3, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732634839643 2024-11-26T15:27:20,904 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 93bcd23a44aa403ba6d02051a98cccbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732634839652 2024-11-26T15:27:20,910 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#B#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:20,910 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/0f1bc001a7594e03825035a2cd4f6316 is 50, key is test_row_0/B:col10/1732634839661/Put/seqid=0 2024-11-26T15:27:20,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742307_1483 (size=12663) 2024-11-26T15:27:20,913 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#A#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:20,913 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/b31a6cb4e8094acaa4c1a576ecfcc233 is 50, key is test_row_0/A:col10/1732634839661/Put/seqid=0 2024-11-26T15:27:20,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742308_1484 (size=12663) 2024-11-26T15:27:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:20,958 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:20,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-26T15:27:20,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/ed78e7b78e0d4b95b6b8047a62f4245d is 50, key is test_row_0/A:col10/1732634839790/Put/seqid=0 2024-11-26T15:27:20,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742309_1485 (size=14541) 2024-11-26T15:27:21,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634900998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:21,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T15:27:21,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:21,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:21,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:21,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T15:27:21,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634901112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:21,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T15:27:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:21,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,317 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/0f1bc001a7594e03825035a2cd4f6316 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0f1bc001a7594e03825035a2cd4f6316 2024-11-26T15:27:21,319 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/b31a6cb4e8094acaa4c1a576ecfcc233 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/b31a6cb4e8094acaa4c1a576ecfcc233 2024-11-26T15:27:21,320 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/B of af3054c22fab85f44f2e3f3ac4020be7 into 0f1bc001a7594e03825035a2cd4f6316(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:21,320 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:21,320 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/B, priority=12, startTime=1732634840898; duration=0sec 2024-11-26T15:27:21,320 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:21,320 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:B 2024-11-26T15:27:21,320 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:27:21,321 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:27:21,321 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/C is initiating minor compaction (all files) 2024-11-26T15:27:21,321 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/C in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:21,321 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/15aa8e57995a4a4aa32e8ef1c14f98b5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/1b1b106ee4714d8da77c3e08a89173b2, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/47b06f17a01c4194a3bbed6e4de669e3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/ff3789757616488d80ac3435ec88a623] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=47.8 K 2024-11-26T15:27:21,321 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 15aa8e57995a4a4aa32e8ef1c14f98b5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732634834956 2024-11-26T15:27:21,322 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b1b106ee4714d8da77c3e08a89173b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732634836252 2024-11-26T15:27:21,322 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 47b06f17a01c4194a3bbed6e4de669e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732634837433 2024-11-26T15:27:21,322 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting ff3789757616488d80ac3435ec88a623, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732634839652 2024-11-26T15:27:21,323 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/A of af3054c22fab85f44f2e3f3ac4020be7 into b31a6cb4e8094acaa4c1a576ecfcc233(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:21,323 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:21,323 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/A, priority=12, startTime=1732634840898; duration=0sec 2024-11-26T15:27:21,323 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:21,323 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:A 2024-11-26T15:27:21,329 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#C#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:21,329 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/72ac4082b9d74c77aee32f9ecd833353 is 50, key is test_row_0/C:col10/1732634839661/Put/seqid=0 2024-11-26T15:27:21,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:21,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634901323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:21,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:21,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:21,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:21,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742310_1486 (size=12663) 2024-11-26T15:27:21,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T15:27:21,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:21,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:21,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:21,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/ed78e7b78e0d4b95b6b8047a62f4245d 2024-11-26T15:27:21,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/d8f5377ef1644aa1ba2c63d0481b9f3a is 50, key is test_row_0/B:col10/1732634839790/Put/seqid=0 2024-11-26T15:27:21,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742311_1487 (size=12151) 2024-11-26T15:27:21,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T15:27:21,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:21,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:21,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:21,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T15:27:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:21,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634901642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:21,736 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/72ac4082b9d74c77aee32f9ecd833353 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/72ac4082b9d74c77aee32f9ecd833353 2024-11-26T15:27:21,740 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/C of af3054c22fab85f44f2e3f3ac4020be7 into 72ac4082b9d74c77aee32f9ecd833353(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:21,740 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:21,740 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/C, priority=12, startTime=1732634840898; duration=0sec 2024-11-26T15:27:21,740 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:21,740 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:C 2024-11-26T15:27:21,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/d8f5377ef1644aa1ba2c63d0481b9f3a 2024-11-26T15:27:21,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/a3f58671fe504a8b8d77dc29073613df is 50, key is test_row_0/C:col10/1732634839790/Put/seqid=0 2024-11-26T15:27:21,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742312_1488 (size=12151) 2024-11-26T15:27:21,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T15:27:21,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:21,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:21,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:21,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T15:27:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:21,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:22,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T15:27:22,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:22,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:22,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:22,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:22,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T15:27:22,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634902152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:22,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/a3f58671fe504a8b8d77dc29073613df 2024-11-26T15:27:22,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/ed78e7b78e0d4b95b6b8047a62f4245d as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ed78e7b78e0d4b95b6b8047a62f4245d 2024-11-26T15:27:22,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ed78e7b78e0d4b95b6b8047a62f4245d, entries=200, sequenceid=243, filesize=14.2 K 2024-11-26T15:27:22,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/d8f5377ef1644aa1ba2c63d0481b9f3a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/d8f5377ef1644aa1ba2c63d0481b9f3a 2024-11-26T15:27:22,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/d8f5377ef1644aa1ba2c63d0481b9f3a, entries=150, sequenceid=243, filesize=11.9 K 2024-11-26T15:27:22,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/a3f58671fe504a8b8d77dc29073613df as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/a3f58671fe504a8b8d77dc29073613df 2024-11-26T15:27:22,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/a3f58671fe504a8b8d77dc29073613df, entries=150, sequenceid=243, filesize=11.9 K 2024-11-26T15:27:22,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for af3054c22fab85f44f2e3f3ac4020be7 in 1249ms, sequenceid=243, compaction requested=false 2024-11-26T15:27:22,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:22,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-26T15:27:22,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:22,253 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-26T15:27:22,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/8e6c3a786adf4ba4aea6fa02bf75180d is 50, key is test_row_0/A:col10/1732634840991/Put/seqid=0 2024-11-26T15:27:22,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742313_1489 (size=12151) 2024-11-26T15:27:22,662 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/8e6c3a786adf4ba4aea6fa02bf75180d 2024-11-26T15:27:22,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/0d0e5d0ffb0c409c9124a075910382aa is 50, key is test_row_0/B:col10/1732634840991/Put/seqid=0 2024-11-26T15:27:22,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742314_1490 (size=12151) 2024-11-26T15:27:23,077 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/0d0e5d0ffb0c409c9124a075910382aa 2024-11-26T15:27:23,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/28f5e9b9b14f42a189b2f85643861c3c is 50, key is test_row_0/C:col10/1732634840991/Put/seqid=0 2024-11-26T15:27:23,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742315_1491 (size=12151) 2024-11-26T15:27:23,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:23,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634903283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:23,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634903392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:23,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:23,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:23,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:23,486 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/28f5e9b9b14f42a189b2f85643861c3c 2024-11-26T15:27:23,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/8e6c3a786adf4ba4aea6fa02bf75180d as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/8e6c3a786adf4ba4aea6fa02bf75180d 2024-11-26T15:27:23,491 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/8e6c3a786adf4ba4aea6fa02bf75180d, entries=150, sequenceid=256, filesize=11.9 K 2024-11-26T15:27:23,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/0d0e5d0ffb0c409c9124a075910382aa as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0d0e5d0ffb0c409c9124a075910382aa 2024-11-26T15:27:23,494 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0d0e5d0ffb0c409c9124a075910382aa, entries=150, sequenceid=256, filesize=11.9 K 2024-11-26T15:27:23,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/28f5e9b9b14f42a189b2f85643861c3c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/28f5e9b9b14f42a189b2f85643861c3c 2024-11-26T15:27:23,497 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/28f5e9b9b14f42a189b2f85643861c3c, entries=150, sequenceid=256, filesize=11.9 K 2024-11-26T15:27:23,498 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for af3054c22fab85f44f2e3f3ac4020be7 in 1245ms, sequenceid=256, compaction requested=true 2024-11-26T15:27:23,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:23,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:23,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-26T15:27:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-26T15:27:23,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-26T15:27:23,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5340 sec 2024-11-26T15:27:23,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 3.5380 sec 2024-11-26T15:27:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:23,606 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:23,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-26T15:27:23,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/c80bfcbbfde34863be3a70a59082f6ed is 50, key is test_row_0/A:col10/1732634843243/Put/seqid=0 2024-11-26T15:27:23,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742316_1492 (size=14741) 2024-11-26T15:27:23,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:23,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634903639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:23,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:23,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:23,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:23,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634903752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:23,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634903963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:24,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/c80bfcbbfde34863be3a70a59082f6ed 2024-11-26T15:27:24,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/95eb9bc1fbeb45ce9e334c8e4e1d6192 is 50, key is test_row_0/B:col10/1732634843243/Put/seqid=0 2024-11-26T15:27:24,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742317_1493 (size=12301) 2024-11-26T15:27:24,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/95eb9bc1fbeb45ce9e334c8e4e1d6192 2024-11-26T15:27:24,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/59b43657005646728c408ec7b3e9972f is 50, key is test_row_0/C:col10/1732634843243/Put/seqid=0 2024-11-26T15:27:24,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742318_1494 (size=12301) 2024-11-26T15:27:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-26T15:27:24,132 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:24,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:27:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-26T15:27:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T15:27:24,134 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:27:24,135 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:27:24,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:27:24,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T15:27:24,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634904272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:24,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T15:27:24,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:24,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:24,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:24,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:24,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:24,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T15:27:24,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:24,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:24,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:24,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:24,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:24,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/59b43657005646728c408ec7b3e9972f 2024-11-26T15:27:24,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/c80bfcbbfde34863be3a70a59082f6ed as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/c80bfcbbfde34863be3a70a59082f6ed 2024-11-26T15:27:24,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/c80bfcbbfde34863be3a70a59082f6ed, entries=200, sequenceid=283, filesize=14.4 K 2024-11-26T15:27:24,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/95eb9bc1fbeb45ce9e334c8e4e1d6192 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/95eb9bc1fbeb45ce9e334c8e4e1d6192 2024-11-26T15:27:24,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/95eb9bc1fbeb45ce9e334c8e4e1d6192, entries=150, sequenceid=283, filesize=12.0 K 2024-11-26T15:27:24,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/59b43657005646728c408ec7b3e9972f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/59b43657005646728c408ec7b3e9972f 2024-11-26T15:27:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T15:27:24,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/59b43657005646728c408ec7b3e9972f, entries=150, sequenceid=283, filesize=12.0 K 2024-11-26T15:27:24,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for af3054c22fab85f44f2e3f3ac4020be7 in 850ms, sequenceid=283, compaction requested=true 2024-11-26T15:27:24,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:24,456 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:27:24,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:27:24,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:24,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:27:24,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:24,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:27:24,457 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:27:24,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:24,457 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54096 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:27:24,458 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/A is initiating minor compaction (all files) 2024-11-26T15:27:24,458 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/A in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:24,458 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/b31a6cb4e8094acaa4c1a576ecfcc233, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ed78e7b78e0d4b95b6b8047a62f4245d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/8e6c3a786adf4ba4aea6fa02bf75180d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/c80bfcbbfde34863be3a70a59082f6ed] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=52.8 K 2024-11-26T15:27:24,458 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:27:24,458 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/B is initiating minor compaction (all files) 2024-11-26T15:27:24,458 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting b31a6cb4e8094acaa4c1a576ecfcc233, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732634839652 2024-11-26T15:27:24,458 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/B in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:24,458 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0f1bc001a7594e03825035a2cd4f6316, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/d8f5377ef1644aa1ba2c63d0481b9f3a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0d0e5d0ffb0c409c9124a075910382aa, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/95eb9bc1fbeb45ce9e334c8e4e1d6192] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=48.1 K 2024-11-26T15:27:24,458 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed78e7b78e0d4b95b6b8047a62f4245d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732634839740 2024-11-26T15:27:24,458 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f1bc001a7594e03825035a2cd4f6316, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732634839652 2024-11-26T15:27:24,459 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e6c3a786adf4ba4aea6fa02bf75180d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732634840968 2024-11-26T15:27:24,459 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting d8f5377ef1644aa1ba2c63d0481b9f3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732634839778 2024-11-26T15:27:24,459 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting c80bfcbbfde34863be3a70a59082f6ed, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732634843243 2024-11-26T15:27:24,459 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d0e5d0ffb0c409c9124a075910382aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732634840968 2024-11-26T15:27:24,459 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 95eb9bc1fbeb45ce9e334c8e4e1d6192, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732634843243 2024-11-26T15:27:24,466 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#B#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:24,467 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/490a0694e4924b11bd1028e342f128c1 is 50, key is test_row_0/B:col10/1732634843243/Put/seqid=0 2024-11-26T15:27:24,474 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#A#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:24,475 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/be773c0dd94745bf87cde52eea9060ed is 50, key is test_row_0/A:col10/1732634843243/Put/seqid=0 2024-11-26T15:27:24,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742319_1495 (size=12949) 2024-11-26T15:27:24,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742320_1496 (size=12949) 2024-11-26T15:27:24,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-26T15:27:24,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:24,593 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-26T15:27:24,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/cd88e0a34ea940aea1e61b2033468002 is 50, key is test_row_0/A:col10/1732634843618/Put/seqid=0 2024-11-26T15:27:24,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742321_1497 (size=12301) 2024-11-26T15:27:24,602 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/cd88e0a34ea940aea1e61b2033468002 2024-11-26T15:27:24,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/06e098226873422399b665827b8f467e is 50, key is test_row_0/B:col10/1732634843618/Put/seqid=0 2024-11-26T15:27:24,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742322_1498 (size=12301) 2024-11-26T15:27:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T15:27:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:24,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:24,881 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/490a0694e4924b11bd1028e342f128c1 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/490a0694e4924b11bd1028e342f128c1 2024-11-26T15:27:24,886 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/be773c0dd94745bf87cde52eea9060ed as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/be773c0dd94745bf87cde52eea9060ed 2024-11-26T15:27:24,886 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/B of af3054c22fab85f44f2e3f3ac4020be7 into 490a0694e4924b11bd1028e342f128c1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:24,886 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:24,886 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/B, priority=12, startTime=1732634844456; duration=0sec 2024-11-26T15:27:24,887 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:24,887 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:B 2024-11-26T15:27:24,887 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:27:24,887 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:27:24,887 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/C is initiating minor compaction (all files) 2024-11-26T15:27:24,887 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/C in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:24,888 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/72ac4082b9d74c77aee32f9ecd833353, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/a3f58671fe504a8b8d77dc29073613df, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/28f5e9b9b14f42a189b2f85643861c3c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/59b43657005646728c408ec7b3e9972f] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=48.1 K 2024-11-26T15:27:24,888 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 72ac4082b9d74c77aee32f9ecd833353, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732634839652 2024-11-26T15:27:24,888 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting a3f58671fe504a8b8d77dc29073613df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732634839778 2024-11-26T15:27:24,888 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 28f5e9b9b14f42a189b2f85643861c3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732634840968 2024-11-26T15:27:24,888 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 59b43657005646728c408ec7b3e9972f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732634843243 2024-11-26T15:27:24,890 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/A of af3054c22fab85f44f2e3f3ac4020be7 into be773c0dd94745bf87cde52eea9060ed(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:24,890 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:24,890 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/A, priority=12, startTime=1732634844456; duration=0sec 2024-11-26T15:27:24,890 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:24,890 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:A 2024-11-26T15:27:24,896 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#C#compaction#423 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:24,896 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/dacd918d84614dcc9b7b9f0760cce766 is 50, key is test_row_0/C:col10/1732634843243/Put/seqid=0 2024-11-26T15:27:24,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742323_1499 (size=12949) 2024-11-26T15:27:24,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634904942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:25,013 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/06e098226873422399b665827b8f467e 2024-11-26T15:27:25,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/66f8e4bd0440410d9e9a36bc8ddedab2 is 50, key is test_row_0/C:col10/1732634843618/Put/seqid=0 2024-11-26T15:27:25,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742324_1500 (size=12301) 2024-11-26T15:27:25,029 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/66f8e4bd0440410d9e9a36bc8ddedab2 2024-11-26T15:27:25,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/cd88e0a34ea940aea1e61b2033468002 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/cd88e0a34ea940aea1e61b2033468002 2024-11-26T15:27:25,037 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/cd88e0a34ea940aea1e61b2033468002, entries=150, sequenceid=292, filesize=12.0 K 2024-11-26T15:27:25,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/06e098226873422399b665827b8f467e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/06e098226873422399b665827b8f467e 2024-11-26T15:27:25,040 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/06e098226873422399b665827b8f467e, entries=150, sequenceid=292, filesize=12.0 K 2024-11-26T15:27:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/66f8e4bd0440410d9e9a36bc8ddedab2 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/66f8e4bd0440410d9e9a36bc8ddedab2 2024-11-26T15:27:25,043 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/66f8e4bd0440410d9e9a36bc8ddedab2, entries=150, sequenceid=292, filesize=12.0 K 2024-11-26T15:27:25,044 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for af3054c22fab85f44f2e3f3ac4020be7 in 451ms, sequenceid=292, compaction requested=false 2024-11-26T15:27:25,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:25,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:25,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-26T15:27:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-26T15:27:25,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-26T15:27:25,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 910 msec 2024-11-26T15:27:25,048 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 914 msec 2024-11-26T15:27:25,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:25,058 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:25,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-26T15:27:25,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/9272568454a0496ea51fe779b4dcb35e is 50, key is test_row_0/A:col10/1732634844930/Put/seqid=0 2024-11-26T15:27:25,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742325_1501 (size=14741) 2024-11-26T15:27:25,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634905088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:25,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:25,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:25,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:25,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634905203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:25,238 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T15:27:25,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T15:27:25,272 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:25,273 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:27:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-26T15:27:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T15:27:25,274 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:27:25,275 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:27:25,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:27:25,307 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/dacd918d84614dcc9b7b9f0760cce766 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/dacd918d84614dcc9b7b9f0760cce766 2024-11-26T15:27:25,310 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/C of af3054c22fab85f44f2e3f3ac4020be7 into dacd918d84614dcc9b7b9f0760cce766(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:25,310 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:25,310 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/C, priority=12, startTime=1732634844457; duration=0sec 2024-11-26T15:27:25,310 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:25,310 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:C 2024-11-26T15:27:25,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T15:27:25,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634905412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:25,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-26T15:27:25,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:25,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:25,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:25,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:25,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:25,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/9272568454a0496ea51fe779b4dcb35e 2024-11-26T15:27:25,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/7b45fa3f41434298865057951a480104 is 50, key is test_row_0/B:col10/1732634844930/Put/seqid=0 2024-11-26T15:27:25,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742326_1502 (size=12301) 2024-11-26T15:27:25,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-26T15:27:25,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:25,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:25,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:25,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:25,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T15:27:25,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634905722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:25,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:25,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:25,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:25,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-26T15:27:25,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:25,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:25,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:25,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:25,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:25,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/7b45fa3f41434298865057951a480104 2024-11-26T15:27:25,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/79008eb52c774e1f9c9b83369e187c5f is 50, key is test_row_0/C:col10/1732634844930/Put/seqid=0 2024-11-26T15:27:25,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742327_1503 (size=12301) 2024-11-26T15:27:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T15:27:25,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-26T15:27:25,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:25,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:25,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:25,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:25,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:25,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:26,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-26T15:27:26,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:26,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:26,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:26,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:26,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:26,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634906233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:26,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:26,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:26,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:26,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-26T15:27:26,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:26,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:26,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:26,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:26,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:26,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/79008eb52c774e1f9c9b83369e187c5f 2024-11-26T15:27:26,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/9272568454a0496ea51fe779b4dcb35e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/9272568454a0496ea51fe779b4dcb35e 2024-11-26T15:27:26,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/9272568454a0496ea51fe779b4dcb35e, entries=200, sequenceid=322, filesize=14.4 K 2024-11-26T15:27:26,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/7b45fa3f41434298865057951a480104 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/7b45fa3f41434298865057951a480104 2024-11-26T15:27:26,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/7b45fa3f41434298865057951a480104, entries=150, sequenceid=322, filesize=12.0 K 2024-11-26T15:27:26,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/79008eb52c774e1f9c9b83369e187c5f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/79008eb52c774e1f9c9b83369e187c5f 2024-11-26T15:27:26,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/79008eb52c774e1f9c9b83369e187c5f, entries=150, sequenceid=322, filesize=12.0 K 2024-11-26T15:27:26,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for af3054c22fab85f44f2e3f3ac4020be7 in 1239ms, sequenceid=322, compaction requested=true 2024-11-26T15:27:26,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:26,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:27:26,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:26,297 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:26,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:27:26,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:26,297 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:26,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:27:26,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:26,298 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:26,298 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:26,298 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/A is initiating minor compaction (all files) 2024-11-26T15:27:26,298 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/B is initiating minor compaction (all files) 2024-11-26T15:27:26,298 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/A in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:26,298 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/B in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:26,298 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/be773c0dd94745bf87cde52eea9060ed, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/cd88e0a34ea940aea1e61b2033468002, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/9272568454a0496ea51fe779b4dcb35e] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=39.1 K 2024-11-26T15:27:26,298 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/490a0694e4924b11bd1028e342f128c1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/06e098226873422399b665827b8f467e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/7b45fa3f41434298865057951a480104] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=36.7 K 2024-11-26T15:27:26,298 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting be773c0dd94745bf87cde52eea9060ed, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732634843243 2024-11-26T15:27:26,298 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 490a0694e4924b11bd1028e342f128c1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732634843243 2024-11-26T15:27:26,298 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd88e0a34ea940aea1e61b2033468002, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732634843614 2024-11-26T15:27:26,298 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 06e098226873422399b665827b8f467e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732634843614 2024-11-26T15:27:26,299 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9272568454a0496ea51fe779b4dcb35e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732634844930 2024-11-26T15:27:26,299 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b45fa3f41434298865057951a480104, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732634844930 2024-11-26T15:27:26,304 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#B#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:26,304 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#A#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:26,305 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/575ebd2cda4b40139338b7fc44e80af1 is 50, key is test_row_0/B:col10/1732634844930/Put/seqid=0 2024-11-26T15:27:26,305 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/6aaa1de8fee041578f4f033f12ff3ec0 is 50, key is test_row_0/A:col10/1732634844930/Put/seqid=0 2024-11-26T15:27:26,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742328_1504 (size=13051) 2024-11-26T15:27:26,315 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/6aaa1de8fee041578f4f033f12ff3ec0 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/6aaa1de8fee041578f4f033f12ff3ec0 2024-11-26T15:27:26,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742329_1505 (size=13051) 2024-11-26T15:27:26,320 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/A of af3054c22fab85f44f2e3f3ac4020be7 into 6aaa1de8fee041578f4f033f12ff3ec0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:26,320 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:26,320 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/A, priority=13, startTime=1732634846297; duration=0sec 2024-11-26T15:27:26,320 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:26,320 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:A 2024-11-26T15:27:26,320 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:26,321 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:26,321 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/C is initiating minor compaction (all files) 2024-11-26T15:27:26,321 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/C in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:26,321 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/dacd918d84614dcc9b7b9f0760cce766, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/66f8e4bd0440410d9e9a36bc8ddedab2, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/79008eb52c774e1f9c9b83369e187c5f] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=36.7 K 2024-11-26T15:27:26,321 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting dacd918d84614dcc9b7b9f0760cce766, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732634843243 2024-11-26T15:27:26,322 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66f8e4bd0440410d9e9a36bc8ddedab2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732634843614 2024-11-26T15:27:26,322 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79008eb52c774e1f9c9b83369e187c5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732634844930 2024-11-26T15:27:26,328 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#C#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:26,328 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/85d5fd778627459791dbd47d05826005 is 50, key is test_row_0/C:col10/1732634844930/Put/seqid=0 2024-11-26T15:27:26,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742330_1506 (size=13051) 2024-11-26T15:27:26,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-26T15:27:26,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:26,392 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-26T15:27:26,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/f88d6edba0c342029f0d2ad268f11702 is 50, key is test_row_0/A:col10/1732634845062/Put/seqid=0 2024-11-26T15:27:26,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742331_1507 (size=12301) 2024-11-26T15:27:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T15:27:26,719 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/575ebd2cda4b40139338b7fc44e80af1 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/575ebd2cda4b40139338b7fc44e80af1 2024-11-26T15:27:26,723 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/B of af3054c22fab85f44f2e3f3ac4020be7 into 575ebd2cda4b40139338b7fc44e80af1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:26,723 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:26,723 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/B, priority=13, startTime=1732634846297; duration=0sec 2024-11-26T15:27:26,723 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:26,724 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:B 2024-11-26T15:27:26,735 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/85d5fd778627459791dbd47d05826005 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/85d5fd778627459791dbd47d05826005 2024-11-26T15:27:26,738 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/C of af3054c22fab85f44f2e3f3ac4020be7 into 85d5fd778627459791dbd47d05826005(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:26,738 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:26,738 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/C, priority=13, startTime=1732634846297; duration=0sec 2024-11-26T15:27:26,738 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:26,738 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:C 2024-11-26T15:27:26,800 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/f88d6edba0c342029f0d2ad268f11702 2024-11-26T15:27:26,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/f231fc8ace3b4e98b8223c08833a64d3 is 50, key is test_row_0/B:col10/1732634845062/Put/seqid=0 2024-11-26T15:27:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:26,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. as already flushing 2024-11-26T15:27:26,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742332_1508 (size=12301) 2024-11-26T15:27:26,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48140 deadline: 1732634906863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:26,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1732634906864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:26,867 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_2' in af3054c22fab85f44f2e3f3ac4020be7 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18312 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:26,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48120 deadline: 1732634906865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:26,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:26,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:26,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:26,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48188 deadline: 1732634906866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:26,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:26,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:26,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:26,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1732634906972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:26,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48120 deadline: 1732634906983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:26,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48188 deadline: 1732634906983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:27,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1732634907182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:27,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:27,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:27,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:27,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48120 deadline: 1732634907192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:27,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48188 deadline: 1732634907192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:27,211 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/f231fc8ace3b4e98b8223c08833a64d3 2024-11-26T15:27:27,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/702494fca88a4bc699363186cdb05d11 is 50, key is test_row_0/C:col10/1732634845062/Put/seqid=0 2024-11-26T15:27:27,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742333_1509 (size=12301) 2024-11-26T15:27:27,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48154 deadline: 1732634907252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:27,256 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in af3054c22fab85f44f2e3f3ac4020be7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:27,341 INFO [Thread-1981 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1981. 2024-11-26T15:27:27,342 DEBUG [Thread-1981 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:27:27,342 DEBUG [Thread-1981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:27,342 DEBUG [Thread-1981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:27,342 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:27:27,346 INFO [Thread-1985 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1985. 2024-11-26T15:27:27,346 DEBUG [Thread-1985 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:27:27,346 DEBUG [Thread-1985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:27,346 DEBUG [Thread-1985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:27,347 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:27:27,348 INFO [Thread-1979 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1979. 2024-11-26T15:27:27,348 DEBUG [Thread-1979 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:27:27,348 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:27,348 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:27,348 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:27:27,349 INFO [Thread-1977 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1977. 2024-11-26T15:27:27,349 DEBUG [Thread-1977 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:27:27,349 DEBUG [Thread-1977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:27,349 DEBUG [Thread-1977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:27,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:27:27,354 INFO [Thread-1983 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1983. 2024-11-26T15:27:27,354 DEBUG [Thread-1983 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:27:27,354 DEBUG [Thread-1983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:27,354 DEBUG [Thread-1983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:27,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:27:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T15:27:27,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48170 deadline: 1732634907493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:27,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:27,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:27,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:27,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48120 deadline: 1732634907503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:27,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48188 deadline: 1732634907503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 , the old value is region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=2 because the exception is null or not the one we care about 2024-11-26T15:27:27,621 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/702494fca88a4bc699363186cdb05d11 2024-11-26T15:27:27,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/f88d6edba0c342029f0d2ad268f11702 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/f88d6edba0c342029f0d2ad268f11702 2024-11-26T15:27:27,636 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/f88d6edba0c342029f0d2ad268f11702, entries=150, sequenceid=332, filesize=12.0 K 2024-11-26T15:27:27,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/f231fc8ace3b4e98b8223c08833a64d3 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/f231fc8ace3b4e98b8223c08833a64d3 2024-11-26T15:27:27,640 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/f231fc8ace3b4e98b8223c08833a64d3, entries=150, sequenceid=332, filesize=12.0 K 2024-11-26T15:27:27,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/702494fca88a4bc699363186cdb05d11 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/702494fca88a4bc699363186cdb05d11 2024-11-26T15:27:27,644 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/702494fca88a4bc699363186cdb05d11, entries=150, sequenceid=332, filesize=12.0 K 2024-11-26T15:27:27,645 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for af3054c22fab85f44f2e3f3ac4020be7 in 1253ms, sequenceid=332, compaction requested=false 2024-11-26T15:27:27,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:27,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:27,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-26T15:27:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-26T15:27:27,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-26T15:27:27,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3710 sec 2024-11-26T15:27:27,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.3750 sec 2024-11-26T15:27:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:28,004 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:28,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-26T15:27:28,004 INFO [Thread-1968 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1968. 2024-11-26T15:27:28,004 DEBUG [Thread-1968 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:27:28,004 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:28,005 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:28,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:27:28,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/9f3f8692e07b41a5b6d067f9c5b5dddd is 50, key is test_row_0/A:col10/1732634846864/Put/seqid=0 2024-11-26T15:27:28,014 INFO [Thread-1970 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1970. 2024-11-26T15:27:28,014 INFO [Thread-1966 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1966. 2024-11-26T15:27:28,014 DEBUG [Thread-1970 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:27:28,014 DEBUG [Thread-1966 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:27:28,014 DEBUG [Thread-1966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:28,014 DEBUG [Thread-1970 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:28,014 DEBUG [Thread-1970 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:28,014 DEBUG [Thread-1966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:28,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:27:28,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:27:28,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742334_1510 (size=12301) 2024-11-26T15:27:28,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/9f3f8692e07b41a5b6d067f9c5b5dddd 2024-11-26T15:27:28,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/35d58d809325465c887a3dd18c839a38 is 50, key is test_row_0/B:col10/1732634846864/Put/seqid=0 2024-11-26T15:27:28,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742335_1511 (size=12301) 2024-11-26T15:27:28,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/35d58d809325465c887a3dd18c839a38 2024-11-26T15:27:28,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/0338d2a67e2443a6920ffdcff3689fb0 is 50, key is test_row_0/C:col10/1732634846864/Put/seqid=0 2024-11-26T15:27:28,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742336_1512 (size=12301) 2024-11-26T15:27:29,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/0338d2a67e2443a6920ffdcff3689fb0 2024-11-26T15:27:29,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/9f3f8692e07b41a5b6d067f9c5b5dddd as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/9f3f8692e07b41a5b6d067f9c5b5dddd 2024-11-26T15:27:29,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/9f3f8692e07b41a5b6d067f9c5b5dddd, entries=150, sequenceid=363, filesize=12.0 K 2024-11-26T15:27:29,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/35d58d809325465c887a3dd18c839a38 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/35d58d809325465c887a3dd18c839a38 2024-11-26T15:27:29,274 INFO [Thread-1974 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1974. 2024-11-26T15:27:29,274 DEBUG [Thread-1974 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:27:29,274 DEBUG [Thread-1974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:29,274 DEBUG [Thread-1974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:29,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:27:29,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/35d58d809325465c887a3dd18c839a38, entries=150, sequenceid=363, filesize=12.0 K 2024-11-26T15:27:29,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/0338d2a67e2443a6920ffdcff3689fb0 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/0338d2a67e2443a6920ffdcff3689fb0 2024-11-26T15:27:29,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/0338d2a67e2443a6920ffdcff3689fb0, entries=150, sequenceid=363, filesize=12.0 K 2024-11-26T15:27:29,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=20.13 KB/20610 for af3054c22fab85f44f2e3f3ac4020be7 in 1277ms, sequenceid=363, compaction requested=true 2024-11-26T15:27:29,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:29,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:27:29,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:29,282 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:29,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:27:29,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:29,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store af3054c22fab85f44f2e3f3ac4020be7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:27:29,282 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:29,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:29,282 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:29,283 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:29,283 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/A is initiating minor compaction (all files) 2024-11-26T15:27:29,283 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/B is initiating minor compaction (all files) 2024-11-26T15:27:29,283 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/A in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:29,283 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/B in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:29,283 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/6aaa1de8fee041578f4f033f12ff3ec0, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/f88d6edba0c342029f0d2ad268f11702, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/9f3f8692e07b41a5b6d067f9c5b5dddd] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=36.8 K 2024-11-26T15:27:29,283 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/575ebd2cda4b40139338b7fc44e80af1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/f231fc8ace3b4e98b8223c08833a64d3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/35d58d809325465c887a3dd18c839a38] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=36.8 K 2024-11-26T15:27:29,283 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 575ebd2cda4b40139338b7fc44e80af1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732634844930 2024-11-26T15:27:29,283 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6aaa1de8fee041578f4f033f12ff3ec0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732634844930 2024-11-26T15:27:29,283 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting f88d6edba0c342029f0d2ad268f11702, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732634845062 2024-11-26T15:27:29,283 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting f231fc8ace3b4e98b8223c08833a64d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732634845062 2024-11-26T15:27:29,284 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 35d58d809325465c887a3dd18c839a38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732634846853 2024-11-26T15:27:29,284 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f3f8692e07b41a5b6d067f9c5b5dddd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732634846853 2024-11-26T15:27:29,293 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#B#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:29,293 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#A#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:29,293 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/c18402b6747742c2bd1c425e4af2fd1b is 50, key is test_row_0/A:col10/1732634846864/Put/seqid=0 2024-11-26T15:27:29,293 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/2469bb8a7479415ba2b6af401d3fff83 is 50, key is test_row_0/B:col10/1732634846864/Put/seqid=0 2024-11-26T15:27:29,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742337_1513 (size=13153) 2024-11-26T15:27:29,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742338_1514 (size=13153) 2024-11-26T15:27:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-26T15:27:29,432 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:29,710 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/c18402b6747742c2bd1c425e4af2fd1b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/c18402b6747742c2bd1c425e4af2fd1b 2024-11-26T15:27:29,710 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/2469bb8a7479415ba2b6af401d3fff83 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/2469bb8a7479415ba2b6af401d3fff83 2024-11-26T15:27:29,715 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/A of af3054c22fab85f44f2e3f3ac4020be7 into c18402b6747742c2bd1c425e4af2fd1b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:29,715 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/B of af3054c22fab85f44f2e3f3ac4020be7 into 2469bb8a7479415ba2b6af401d3fff83(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:29,715 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:29,715 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:29,715 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/A, priority=13, startTime=1732634849281; duration=0sec 2024-11-26T15:27:29,715 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/B, priority=13, startTime=1732634849282; duration=0sec 2024-11-26T15:27:29,715 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:29,715 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:29,715 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:A 2024-11-26T15:27:29,715 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:B 2024-11-26T15:27:29,715 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:29,716 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:29,716 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): af3054c22fab85f44f2e3f3ac4020be7/C is initiating minor compaction (all files) 2024-11-26T15:27:29,716 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of af3054c22fab85f44f2e3f3ac4020be7/C in TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:29,716 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/85d5fd778627459791dbd47d05826005, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/702494fca88a4bc699363186cdb05d11, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/0338d2a67e2443a6920ffdcff3689fb0] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp, totalSize=36.8 K 2024-11-26T15:27:29,716 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85d5fd778627459791dbd47d05826005, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732634844930 2024-11-26T15:27:29,717 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 702494fca88a4bc699363186cdb05d11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732634845062 2024-11-26T15:27:29,717 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0338d2a67e2443a6920ffdcff3689fb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732634846853 2024-11-26T15:27:29,725 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): af3054c22fab85f44f2e3f3ac4020be7#C#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:29,725 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/501f9a3ce25a4e59b3dcd9eb4806dca6 is 50, key is test_row_0/C:col10/1732634846864/Put/seqid=0 2024-11-26T15:27:29,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742339_1515 (size=13153) 2024-11-26T15:27:30,141 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/501f9a3ce25a4e59b3dcd9eb4806dca6 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/501f9a3ce25a4e59b3dcd9eb4806dca6 2024-11-26T15:27:30,147 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in af3054c22fab85f44f2e3f3ac4020be7/C of af3054c22fab85f44f2e3f3ac4020be7 into 501f9a3ce25a4e59b3dcd9eb4806dca6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:30,147 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for af3054c22fab85f44f2e3f3ac4020be7: 2024-11-26T15:27:30,147 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., storeName=af3054c22fab85f44f2e3f3ac4020be7/C, priority=13, startTime=1732634849282; duration=0sec 2024-11-26T15:27:30,147 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:30,147 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: af3054c22fab85f44f2e3f3ac4020be7:C 2024-11-26T15:27:36,976 INFO [Thread-1972 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1972. 2024-11-26T15:27:36,977 DEBUG [Thread-1972 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:27:36,977 DEBUG [Thread-1972 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:36,977 DEBUG [Thread-1972 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:36,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:27:36,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T15:27:36,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-26T15:27:36,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-26T15:27:36,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-26T15:27:36,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-26T15:27:36,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 229 2024-11-26T15:27:36,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T15:27:36,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T15:27:36,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1636 2024-11-26T15:27:36,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4907 rows 2024-11-26T15:27:36,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1635 2024-11-26T15:27:36,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4901 rows 2024-11-26T15:27:36,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1635 2024-11-26T15:27:36,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4903 rows 2024-11-26T15:27:36,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1632 2024-11-26T15:27:36,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4895 rows 2024-11-26T15:27:36,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1637 2024-11-26T15:27:36,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4908 rows 2024-11-26T15:27:36,979 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T15:27:36,980 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T15:27:36,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:36,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:36,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:27:36,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T15:27:36,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-26T15:27:36,987 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634856987"}]},"ts":"1732634856987"} 2024-11-26T15:27:36,989 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T15:27:36,989 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T15:27:36,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T15:27:36,991 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=af3054c22fab85f44f2e3f3ac4020be7, UNASSIGN}] 2024-11-26T15:27:36,992 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=af3054c22fab85f44f2e3f3ac4020be7, UNASSIGN 2024-11-26T15:27:36,992 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=af3054c22fab85f44f2e3f3ac4020be7, regionState=CLOSING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:36,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=af3054c22fab85f44f2e3f3ac4020be7, UNASSIGN because future has completed 2024-11-26T15:27:36,994 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T15:27:36,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:27:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-26T15:27:37,147 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:37,147 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T15:27:37,148 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing af3054c22fab85f44f2e3f3ac4020be7, disabling compactions & flushes 2024-11-26T15:27:37,148 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:37,148 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:37,148 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. after waiting 0 ms 2024-11-26T15:27:37,148 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:37,148 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing af3054c22fab85f44f2e3f3ac4020be7 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-26T15:27:37,156 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/b2ee85d294854165b256f21058448f6f is 50, key is test_row_0/A:col10/1732634848013/Put/seqid=0 2024-11-26T15:27:37,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742340_1516 (size=12301) 2024-11-26T15:27:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-26T15:27:37,564 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/b2ee85d294854165b256f21058448f6f 2024-11-26T15:27:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-26T15:27:37,677 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/40bf8faa957f47e2aa7a4044cecf22cc is 50, key is test_row_0/B:col10/1732634848013/Put/seqid=0 2024-11-26T15:27:37,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742341_1517 (size=12301) 2024-11-26T15:27:38,083 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/40bf8faa957f47e2aa7a4044cecf22cc 2024-11-26T15:27:38,096 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/42d3a443e9d44159888851fa3a0d4500 is 50, key is test_row_0/C:col10/1732634848013/Put/seqid=0 2024-11-26T15:27:38,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742342_1518 (size=12301) 2024-11-26T15:27:38,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-26T15:27:38,502 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/42d3a443e9d44159888851fa3a0d4500 2024-11-26T15:27:38,513 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/A/b2ee85d294854165b256f21058448f6f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/b2ee85d294854165b256f21058448f6f 2024-11-26T15:27:38,516 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/b2ee85d294854165b256f21058448f6f, entries=150, sequenceid=373, filesize=12.0 K 2024-11-26T15:27:38,517 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/B/40bf8faa957f47e2aa7a4044cecf22cc as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/40bf8faa957f47e2aa7a4044cecf22cc 2024-11-26T15:27:38,520 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/40bf8faa957f47e2aa7a4044cecf22cc, entries=150, sequenceid=373, filesize=12.0 K 2024-11-26T15:27:38,521 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/.tmp/C/42d3a443e9d44159888851fa3a0d4500 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/42d3a443e9d44159888851fa3a0d4500 2024-11-26T15:27:38,524 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/42d3a443e9d44159888851fa3a0d4500, entries=150, sequenceid=373, filesize=12.0 K 2024-11-26T15:27:38,525 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for af3054c22fab85f44f2e3f3ac4020be7 in 1377ms, sequenceid=373, compaction requested=false 2024-11-26T15:27:38,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/e3b922e6ccc2439bb3f5a598f943ecdf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/3cb8afffbf9b464b9de5ddf5e04dc9eb, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ddc9ce54522d4270bcc6aea7d039115b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/6691ea8ef777474dbe517460cc90dd2e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/16d2c6491fee40989d02af38f4a9539f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/7f06f48cc22249bc88dd997907309403, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/92e52de0b319462fb825c3b3f2948f93, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/96807eae4599410f9a73a691dab165b6, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/afeeb0eafa434beb8fde2551bd290d2e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/f2b43c1e31254c20a82211c68c363f55, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/b39d5e9388394e3f9eea390354d7aae4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/cf0f2e2c3cbd4ca5875ad19016ffe1c4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/625cd03d12b7475b86a97bb61cff7089, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/304b88f8b5c24c75a71b35cfcfb50b00, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ea7e699ada844224900243d7ce9213e3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/b31a6cb4e8094acaa4c1a576ecfcc233, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ed78e7b78e0d4b95b6b8047a62f4245d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/8e6c3a786adf4ba4aea6fa02bf75180d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/c80bfcbbfde34863be3a70a59082f6ed, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/be773c0dd94745bf87cde52eea9060ed, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/cd88e0a34ea940aea1e61b2033468002, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/9272568454a0496ea51fe779b4dcb35e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/6aaa1de8fee041578f4f033f12ff3ec0, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/f88d6edba0c342029f0d2ad268f11702, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/9f3f8692e07b41a5b6d067f9c5b5dddd] to archive 2024-11-26T15:27:38,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:27:38,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/e3b922e6ccc2439bb3f5a598f943ecdf to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/e3b922e6ccc2439bb3f5a598f943ecdf 2024-11-26T15:27:38,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/3cb8afffbf9b464b9de5ddf5e04dc9eb to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/3cb8afffbf9b464b9de5ddf5e04dc9eb 2024-11-26T15:27:38,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ddc9ce54522d4270bcc6aea7d039115b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ddc9ce54522d4270bcc6aea7d039115b 2024-11-26T15:27:38,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/6691ea8ef777474dbe517460cc90dd2e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/6691ea8ef777474dbe517460cc90dd2e 2024-11-26T15:27:38,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/16d2c6491fee40989d02af38f4a9539f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/16d2c6491fee40989d02af38f4a9539f 2024-11-26T15:27:38,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/7f06f48cc22249bc88dd997907309403 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/7f06f48cc22249bc88dd997907309403 2024-11-26T15:27:38,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/92e52de0b319462fb825c3b3f2948f93 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/92e52de0b319462fb825c3b3f2948f93 2024-11-26T15:27:38,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/96807eae4599410f9a73a691dab165b6 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/96807eae4599410f9a73a691dab165b6 2024-11-26T15:27:38,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/afeeb0eafa434beb8fde2551bd290d2e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/afeeb0eafa434beb8fde2551bd290d2e 2024-11-26T15:27:38,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/f2b43c1e31254c20a82211c68c363f55 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/f2b43c1e31254c20a82211c68c363f55 2024-11-26T15:27:38,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/b39d5e9388394e3f9eea390354d7aae4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/b39d5e9388394e3f9eea390354d7aae4 2024-11-26T15:27:38,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/cf0f2e2c3cbd4ca5875ad19016ffe1c4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/cf0f2e2c3cbd4ca5875ad19016ffe1c4 2024-11-26T15:27:38,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/625cd03d12b7475b86a97bb61cff7089 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/625cd03d12b7475b86a97bb61cff7089 2024-11-26T15:27:38,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/304b88f8b5c24c75a71b35cfcfb50b00 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/304b88f8b5c24c75a71b35cfcfb50b00 2024-11-26T15:27:38,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ea7e699ada844224900243d7ce9213e3 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ea7e699ada844224900243d7ce9213e3 2024-11-26T15:27:38,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/b31a6cb4e8094acaa4c1a576ecfcc233 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/b31a6cb4e8094acaa4c1a576ecfcc233 2024-11-26T15:27:38,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ed78e7b78e0d4b95b6b8047a62f4245d to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/ed78e7b78e0d4b95b6b8047a62f4245d 2024-11-26T15:27:38,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/8e6c3a786adf4ba4aea6fa02bf75180d to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/8e6c3a786adf4ba4aea6fa02bf75180d 2024-11-26T15:27:38,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/c80bfcbbfde34863be3a70a59082f6ed to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/c80bfcbbfde34863be3a70a59082f6ed 2024-11-26T15:27:38,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/be773c0dd94745bf87cde52eea9060ed to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/be773c0dd94745bf87cde52eea9060ed 2024-11-26T15:27:38,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/cd88e0a34ea940aea1e61b2033468002 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/cd88e0a34ea940aea1e61b2033468002 2024-11-26T15:27:38,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/9272568454a0496ea51fe779b4dcb35e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/9272568454a0496ea51fe779b4dcb35e 2024-11-26T15:27:38,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/6aaa1de8fee041578f4f033f12ff3ec0 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/6aaa1de8fee041578f4f033f12ff3ec0 2024-11-26T15:27:38,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/f88d6edba0c342029f0d2ad268f11702 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/f88d6edba0c342029f0d2ad268f11702 2024-11-26T15:27:38,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/9f3f8692e07b41a5b6d067f9c5b5dddd to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/9f3f8692e07b41a5b6d067f9c5b5dddd 2024-11-26T15:27:38,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a1c929e9ac4a4258907babdc44fac295, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0496c00f152a4c27a1a6db885fd7cbe3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/7e0e2727cc0046659f7a636995b8ec2f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/338bf7a3d4ac434184fa4e87c957cc7d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/63743d7ca05840a890c2fd122f4bed7f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a4a8c5924d4b4200832a245c5ed4260f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/721a1958ba12466f9d85e39e87ec948f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/baaece080dc9445f9354013d0da6ed7a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/8ab9194cdef94a259e05b5099aa3169e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/c48cf3242d57440dbc14d2b3c0647f03, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/201e8b53597d4a52ab475325aced99e7, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/42077007442143e4808c81a9bbfef024, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a8b91bb4dfb047d68d59b5af58c7e581, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/5d91d10021bf45eabcab4af357eb7911, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0f1bc001a7594e03825035a2cd4f6316, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/93bcd23a44aa403ba6d02051a98cccbf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/d8f5377ef1644aa1ba2c63d0481b9f3a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0d0e5d0ffb0c409c9124a075910382aa, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/490a0694e4924b11bd1028e342f128c1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/95eb9bc1fbeb45ce9e334c8e4e1d6192, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/06e098226873422399b665827b8f467e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/575ebd2cda4b40139338b7fc44e80af1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/7b45fa3f41434298865057951a480104, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/f231fc8ace3b4e98b8223c08833a64d3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/35d58d809325465c887a3dd18c839a38] to archive 2024-11-26T15:27:38,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:27:38,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a1c929e9ac4a4258907babdc44fac295 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a1c929e9ac4a4258907babdc44fac295 2024-11-26T15:27:38,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0496c00f152a4c27a1a6db885fd7cbe3 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0496c00f152a4c27a1a6db885fd7cbe3 2024-11-26T15:27:38,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/7e0e2727cc0046659f7a636995b8ec2f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/7e0e2727cc0046659f7a636995b8ec2f 2024-11-26T15:27:38,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/338bf7a3d4ac434184fa4e87c957cc7d to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/338bf7a3d4ac434184fa4e87c957cc7d 2024-11-26T15:27:38,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/63743d7ca05840a890c2fd122f4bed7f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/63743d7ca05840a890c2fd122f4bed7f 2024-11-26T15:27:38,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a4a8c5924d4b4200832a245c5ed4260f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a4a8c5924d4b4200832a245c5ed4260f 2024-11-26T15:27:38,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/721a1958ba12466f9d85e39e87ec948f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/721a1958ba12466f9d85e39e87ec948f 2024-11-26T15:27:38,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/baaece080dc9445f9354013d0da6ed7a to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/baaece080dc9445f9354013d0da6ed7a 2024-11-26T15:27:38,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/8ab9194cdef94a259e05b5099aa3169e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/8ab9194cdef94a259e05b5099aa3169e 2024-11-26T15:27:38,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/c48cf3242d57440dbc14d2b3c0647f03 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/c48cf3242d57440dbc14d2b3c0647f03 2024-11-26T15:27:38,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/201e8b53597d4a52ab475325aced99e7 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/201e8b53597d4a52ab475325aced99e7 2024-11-26T15:27:38,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/42077007442143e4808c81a9bbfef024 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/42077007442143e4808c81a9bbfef024 2024-11-26T15:27:38,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a8b91bb4dfb047d68d59b5af58c7e581 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/a8b91bb4dfb047d68d59b5af58c7e581 2024-11-26T15:27:38,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/5d91d10021bf45eabcab4af357eb7911 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/5d91d10021bf45eabcab4af357eb7911 2024-11-26T15:27:38,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0f1bc001a7594e03825035a2cd4f6316 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0f1bc001a7594e03825035a2cd4f6316 2024-11-26T15:27:38,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/93bcd23a44aa403ba6d02051a98cccbf to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/93bcd23a44aa403ba6d02051a98cccbf 2024-11-26T15:27:38,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/d8f5377ef1644aa1ba2c63d0481b9f3a to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/d8f5377ef1644aa1ba2c63d0481b9f3a 2024-11-26T15:27:38,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0d0e5d0ffb0c409c9124a075910382aa to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/0d0e5d0ffb0c409c9124a075910382aa 2024-11-26T15:27:38,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/490a0694e4924b11bd1028e342f128c1 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/490a0694e4924b11bd1028e342f128c1 2024-11-26T15:27:38,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/95eb9bc1fbeb45ce9e334c8e4e1d6192 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/95eb9bc1fbeb45ce9e334c8e4e1d6192 2024-11-26T15:27:38,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/06e098226873422399b665827b8f467e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/06e098226873422399b665827b8f467e 2024-11-26T15:27:38,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/575ebd2cda4b40139338b7fc44e80af1 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/575ebd2cda4b40139338b7fc44e80af1 2024-11-26T15:27:38,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/7b45fa3f41434298865057951a480104 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/7b45fa3f41434298865057951a480104 2024-11-26T15:27:38,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/f231fc8ace3b4e98b8223c08833a64d3 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/f231fc8ace3b4e98b8223c08833a64d3 2024-11-26T15:27:38,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/35d58d809325465c887a3dd18c839a38 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/35d58d809325465c887a3dd18c839a38 2024-11-26T15:27:38,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/041e7f5faa3c4c5f94dacc12d645908a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/c4bfe7db581142f699cd4380031e9c19, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/671e6aa921a64b1e9d3039ad6be739a2, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/86d93f5f184845a185dad5d9b80f6d08, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/42b3d987cbea4686a9a056aeb327f1d1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/c75a79c25ccd4dbb8c54546fefb66f32, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/2550df2870df4125a92832e20b295bc5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/2e1f80623b2f4a7c94088b58746ec8b5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/de076ff6ed9d480a92ff8d40957be1ad, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/4c244f684e1b4ca2af79a803eea61a94, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/15aa8e57995a4a4aa32e8ef1c14f98b5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/fc29c0a4cd174319bae4b71c0cfcf4b1, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/1b1b106ee4714d8da77c3e08a89173b2, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/47b06f17a01c4194a3bbed6e4de669e3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/72ac4082b9d74c77aee32f9ecd833353, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/ff3789757616488d80ac3435ec88a623, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/a3f58671fe504a8b8d77dc29073613df, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/28f5e9b9b14f42a189b2f85643861c3c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/dacd918d84614dcc9b7b9f0760cce766, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/59b43657005646728c408ec7b3e9972f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/66f8e4bd0440410d9e9a36bc8ddedab2, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/85d5fd778627459791dbd47d05826005, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/79008eb52c774e1f9c9b83369e187c5f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/702494fca88a4bc699363186cdb05d11, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/0338d2a67e2443a6920ffdcff3689fb0] to archive 2024-11-26T15:27:38,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:27:38,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/041e7f5faa3c4c5f94dacc12d645908a to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/041e7f5faa3c4c5f94dacc12d645908a 2024-11-26T15:27:38,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/c4bfe7db581142f699cd4380031e9c19 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/c4bfe7db581142f699cd4380031e9c19 2024-11-26T15:27:38,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/671e6aa921a64b1e9d3039ad6be739a2 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/671e6aa921a64b1e9d3039ad6be739a2 2024-11-26T15:27:38,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/86d93f5f184845a185dad5d9b80f6d08 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/86d93f5f184845a185dad5d9b80f6d08 2024-11-26T15:27:38,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/42b3d987cbea4686a9a056aeb327f1d1 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/42b3d987cbea4686a9a056aeb327f1d1 2024-11-26T15:27:38,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/c75a79c25ccd4dbb8c54546fefb66f32 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/c75a79c25ccd4dbb8c54546fefb66f32 2024-11-26T15:27:38,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/2550df2870df4125a92832e20b295bc5 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/2550df2870df4125a92832e20b295bc5 2024-11-26T15:27:38,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/2e1f80623b2f4a7c94088b58746ec8b5 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/2e1f80623b2f4a7c94088b58746ec8b5 2024-11-26T15:27:38,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/de076ff6ed9d480a92ff8d40957be1ad to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/de076ff6ed9d480a92ff8d40957be1ad 2024-11-26T15:27:38,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/4c244f684e1b4ca2af79a803eea61a94 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/4c244f684e1b4ca2af79a803eea61a94 2024-11-26T15:27:38,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/15aa8e57995a4a4aa32e8ef1c14f98b5 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/15aa8e57995a4a4aa32e8ef1c14f98b5 2024-11-26T15:27:38,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/fc29c0a4cd174319bae4b71c0cfcf4b1 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/fc29c0a4cd174319bae4b71c0cfcf4b1 2024-11-26T15:27:38,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/1b1b106ee4714d8da77c3e08a89173b2 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/1b1b106ee4714d8da77c3e08a89173b2 2024-11-26T15:27:38,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/47b06f17a01c4194a3bbed6e4de669e3 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/47b06f17a01c4194a3bbed6e4de669e3 2024-11-26T15:27:38,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/72ac4082b9d74c77aee32f9ecd833353 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/72ac4082b9d74c77aee32f9ecd833353 2024-11-26T15:27:38,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/ff3789757616488d80ac3435ec88a623 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/ff3789757616488d80ac3435ec88a623 2024-11-26T15:27:38,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/a3f58671fe504a8b8d77dc29073613df to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/a3f58671fe504a8b8d77dc29073613df 2024-11-26T15:27:38,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/28f5e9b9b14f42a189b2f85643861c3c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/28f5e9b9b14f42a189b2f85643861c3c 2024-11-26T15:27:38,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/dacd918d84614dcc9b7b9f0760cce766 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/dacd918d84614dcc9b7b9f0760cce766 2024-11-26T15:27:38,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/59b43657005646728c408ec7b3e9972f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/59b43657005646728c408ec7b3e9972f 2024-11-26T15:27:38,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/66f8e4bd0440410d9e9a36bc8ddedab2 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/66f8e4bd0440410d9e9a36bc8ddedab2 2024-11-26T15:27:38,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/85d5fd778627459791dbd47d05826005 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/85d5fd778627459791dbd47d05826005 2024-11-26T15:27:38,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/79008eb52c774e1f9c9b83369e187c5f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/79008eb52c774e1f9c9b83369e187c5f 2024-11-26T15:27:38,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/702494fca88a4bc699363186cdb05d11 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/702494fca88a4bc699363186cdb05d11 2024-11-26T15:27:38,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/0338d2a67e2443a6920ffdcff3689fb0 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/0338d2a67e2443a6920ffdcff3689fb0 2024-11-26T15:27:38,583 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/recovered.edits/376.seqid, newMaxSeqId=376, maxSeqId=1 2024-11-26T15:27:38,584 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. 2024-11-26T15:27:38,584 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for af3054c22fab85f44f2e3f3ac4020be7: Waiting for close lock at 1732634857147Running coprocessor pre-close hooks at 1732634857147Disabling compacts and flushes for region at 1732634857147Disabling writes for close at 1732634857148 (+1 ms)Obtaining lock to block concurrent updates at 1732634857148Preparing flush snapshotting stores in af3054c22fab85f44f2e3f3ac4020be7 at 1732634857148Finished memstore snapshotting TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732634857149 (+1 ms)Flushing stores of TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7. at 1732634857150 (+1 ms)Flushing af3054c22fab85f44f2e3f3ac4020be7/A: creating writer at 1732634857150Flushing af3054c22fab85f44f2e3f3ac4020be7/A: appending metadata at 1732634857155 (+5 ms)Flushing af3054c22fab85f44f2e3f3ac4020be7/A: closing flushed file at 1732634857155Flushing af3054c22fab85f44f2e3f3ac4020be7/B: creating writer at 1732634857672 (+517 ms)Flushing af3054c22fab85f44f2e3f3ac4020be7/B: appending metadata at 1732634857676 (+4 ms)Flushing af3054c22fab85f44f2e3f3ac4020be7/B: closing flushed file at 1732634857676Flushing af3054c22fab85f44f2e3f3ac4020be7/C: creating writer at 1732634858092 (+416 ms)Flushing af3054c22fab85f44f2e3f3ac4020be7/C: appending metadata at 1732634858096 (+4 ms)Flushing af3054c22fab85f44f2e3f3ac4020be7/C: closing flushed file at 1732634858096Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bc8e297: reopening flushed file at 1732634858512 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70187562: reopening flushed file at 1732634858516 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fe9790b: reopening flushed file at 1732634858520 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for af3054c22fab85f44f2e3f3ac4020be7 in 1377ms, sequenceid=373, compaction requested=false at 1732634858525 (+5 ms)Writing region close event to WAL at 1732634858581 (+56 ms)Running coprocessor post-close hooks at 1732634858583 (+2 ms)Closed at 1732634858584 (+1 ms) 2024-11-26T15:27:38,585 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:38,586 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=af3054c22fab85f44f2e3f3ac4020be7, regionState=CLOSED 2024-11-26T15:27:38,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:27:38,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-26T15:27:38,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure af3054c22fab85f44f2e3f3ac4020be7, server=6cc2bc8bf733,45729,1732634697146 in 1.5930 sec 2024-11-26T15:27:38,590 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-26T15:27:38,590 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=af3054c22fab85f44f2e3f3ac4020be7, UNASSIGN in 1.5970 sec 2024-11-26T15:27:38,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-26T15:27:38,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6010 sec 2024-11-26T15:27:38,592 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634858592"}]},"ts":"1732634858592"} 2024-11-26T15:27:38,593 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T15:27:38,593 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T15:27:38,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6110 sec 2024-11-26T15:27:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-26T15:27:39,134 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:39,135 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T15:27:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:39,140 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-26T15:27:39,142 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:39,146 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:39,149 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/recovered.edits] 2024-11-26T15:27:39,152 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/b2ee85d294854165b256f21058448f6f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/b2ee85d294854165b256f21058448f6f 2024-11-26T15:27:39,154 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/c18402b6747742c2bd1c425e4af2fd1b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/A/c18402b6747742c2bd1c425e4af2fd1b 2024-11-26T15:27:39,158 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/2469bb8a7479415ba2b6af401d3fff83 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/2469bb8a7479415ba2b6af401d3fff83 2024-11-26T15:27:39,159 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/40bf8faa957f47e2aa7a4044cecf22cc to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/B/40bf8faa957f47e2aa7a4044cecf22cc 2024-11-26T15:27:39,162 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/42d3a443e9d44159888851fa3a0d4500 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/42d3a443e9d44159888851fa3a0d4500 2024-11-26T15:27:39,164 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/501f9a3ce25a4e59b3dcd9eb4806dca6 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/C/501f9a3ce25a4e59b3dcd9eb4806dca6 2024-11-26T15:27:39,167 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/recovered.edits/376.seqid to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7/recovered.edits/376.seqid 2024-11-26T15:27:39,168 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/af3054c22fab85f44f2e3f3ac4020be7 2024-11-26T15:27:39,168 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T15:27:39,171 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:39,173 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T15:27:39,175 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T15:27:39,177 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:39,177 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T15:27:39,177 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732634859177"}]},"ts":"9223372036854775807"} 2024-11-26T15:27:39,179 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T15:27:39,179 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => af3054c22fab85f44f2e3f3ac4020be7, NAME => 'TestAcidGuarantees,,1732634825144.af3054c22fab85f44f2e3f3ac4020be7.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T15:27:39,179 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T15:27:39,179 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732634859179"}]},"ts":"9223372036854775807"} 2024-11-26T15:27:39,181 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T15:27:39,182 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:39,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-26T15:27:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-26T15:27:39,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T15:27:39,256 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:39,266 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237 (was 238), OpenFileDescriptor=451 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=376 (was 461), ProcessCount=11 (was 11), AvailableMemoryMB=9257 (was 9322) 2024-11-26T15:27:39,277 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=376, ProcessCount=11, AvailableMemoryMB=9256 2024-11-26T15:27:39,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T15:27:39,279 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T15:27:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:39,280 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T15:27:39,280 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:39,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-26T15:27:39,281 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T15:27:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T15:27:39,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742343_1519 (size=959) 2024-11-26T15:27:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T15:27:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T15:27:39,691 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6 2024-11-26T15:27:39,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742344_1520 (size=53) 2024-11-26T15:27:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T15:27:40,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:27:40,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ade03e3421a188de043c46d5d985046e, disabling compactions & flushes 2024-11-26T15:27:40,103 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:40,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:40,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. after waiting 0 ms 2024-11-26T15:27:40,104 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:40,104 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:40,104 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ade03e3421a188de043c46d5d985046e: Waiting for close lock at 1732634860103Disabling compacts and flushes for region at 1732634860103Disabling writes for close at 1732634860103Writing region close event to WAL at 1732634860104 (+1 ms)Closed at 1732634860104 2024-11-26T15:27:40,107 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T15:27:40,107 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732634860107"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732634860107"}]},"ts":"1732634860107"} 2024-11-26T15:27:40,111 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T15:27:40,113 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T15:27:40,114 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634860113"}]},"ts":"1732634860113"} 2024-11-26T15:27:40,116 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T15:27:40,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ade03e3421a188de043c46d5d985046e, ASSIGN}] 2024-11-26T15:27:40,117 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ade03e3421a188de043c46d5d985046e, ASSIGN 2024-11-26T15:27:40,118 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ade03e3421a188de043c46d5d985046e, ASSIGN; state=OFFLINE, location=6cc2bc8bf733,45729,1732634697146; forceNewPlan=false, retain=false 2024-11-26T15:27:40,269 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=ade03e3421a188de043c46d5d985046e, regionState=OPENING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:40,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ade03e3421a188de043c46d5d985046e, ASSIGN because future has completed 2024-11-26T15:27:40,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:27:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T15:27:40,434 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:40,434 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:27:40,435 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:40,435 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:27:40,435 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:40,436 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:40,438 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:40,440 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ade03e3421a188de043c46d5d985046e columnFamilyName A 2024-11-26T15:27:40,440 DEBUG [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:40,441 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] regionserver.HStore(327): Store=ade03e3421a188de043c46d5d985046e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:27:40,441 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:40,443 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ade03e3421a188de043c46d5d985046e columnFamilyName B 2024-11-26T15:27:40,443 DEBUG [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:40,443 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] regionserver.HStore(327): Store=ade03e3421a188de043c46d5d985046e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:27:40,444 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:40,445 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ade03e3421a188de043c46d5d985046e columnFamilyName C 2024-11-26T15:27:40,445 DEBUG [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:40,446 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] regionserver.HStore(327): Store=ade03e3421a188de043c46d5d985046e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:27:40,446 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:40,447 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:40,447 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:40,448 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:40,449 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:40,449 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T15:27:40,451 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:40,453 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:27:40,453 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened ade03e3421a188de043c46d5d985046e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61052072, jitterRate=-0.09025323390960693}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T15:27:40,453 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:40,454 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for ade03e3421a188de043c46d5d985046e: Running coprocessor pre-open hook at 1732634860436Writing region info on filesystem at 1732634860436Initializing all the Stores at 1732634860437 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634860438 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634860438Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634860438Cleaning up temporary data from old regions at 1732634860449 (+11 ms)Running coprocessor post-open hooks at 1732634860453 (+4 ms)Region opened successfully at 1732634860454 (+1 ms) 2024-11-26T15:27:40,455 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., pid=142, masterSystemTime=1732634860428 2024-11-26T15:27:40,457 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:40,457 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:40,457 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=ade03e3421a188de043c46d5d985046e, regionState=OPEN, openSeqNum=2, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:40,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:27:40,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-26T15:27:40,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 in 186 msec 2024-11-26T15:27:40,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-26T15:27:40,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ade03e3421a188de043c46d5d985046e, ASSIGN in 346 msec 2024-11-26T15:27:40,465 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T15:27:40,465 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634860465"}]},"ts":"1732634860465"} 2024-11-26T15:27:40,467 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T15:27:40,468 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T15:27:40,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1890 sec 2024-11-26T15:27:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T15:27:41,434 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:41,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aaec12e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:41,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:41,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:41,440 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b87377, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:41,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:41,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:41,443 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:41,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73473f05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:41,444 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:41,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:41,447 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T15:27:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 6cc2bc8bf733,37081,1732634696319 2024-11-26T15:27:41,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7a579a95 2024-11-26T15:27:41,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T15:27:41,450 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T15:27:41,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T15:27:41,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T15:27:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T15:27:41,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742345_1521 (size=995) 2024-11-26T15:27:41,868 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-26T15:27:41,868 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-26T15:27:41,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T15:27:41,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ade03e3421a188de043c46d5d985046e, REOPEN/MOVE}] 2024-11-26T15:27:41,879 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ade03e3421a188de043c46d5d985046e, REOPEN/MOVE 2024-11-26T15:27:41,881 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=ade03e3421a188de043c46d5d985046e, regionState=CLOSING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:41,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ade03e3421a188de043c46d5d985046e, REOPEN/MOVE because future has completed 2024-11-26T15:27:41,883 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T15:27:41,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:27:42,036 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,036 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T15:27:42,036 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing ade03e3421a188de043c46d5d985046e, disabling compactions & flushes 2024-11-26T15:27:42,036 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:42,036 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:42,036 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. after waiting 0 ms 2024-11-26T15:27:42,036 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:42,040 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T15:27:42,041 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:42,041 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for ade03e3421a188de043c46d5d985046e: Waiting for close lock at 1732634862036Running coprocessor pre-close hooks at 1732634862036Disabling compacts and flushes for region at 1732634862036Disabling writes for close at 1732634862036Writing region close event to WAL at 1732634862037 (+1 ms)Running coprocessor post-close hooks at 1732634862041 (+4 ms)Closed at 1732634862041 2024-11-26T15:27:42,041 WARN [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: ade03e3421a188de043c46d5d985046e to self. 2024-11-26T15:27:42,042 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,043 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=ade03e3421a188de043c46d5d985046e, regionState=CLOSED 2024-11-26T15:27:42,044 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:27:42,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-26T15:27:42,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 in 161 msec 2024-11-26T15:27:42,046 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ade03e3421a188de043c46d5d985046e, REOPEN/MOVE; state=CLOSED, location=6cc2bc8bf733,45729,1732634697146; forceNewPlan=false, retain=true 2024-11-26T15:27:42,197 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=ade03e3421a188de043c46d5d985046e, regionState=OPENING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ade03e3421a188de043c46d5d985046e, REOPEN/MOVE because future has completed 2024-11-26T15:27:42,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:27:42,355 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:42,355 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:27:42,356 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,356 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:27:42,356 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,356 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,359 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,360 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ade03e3421a188de043c46d5d985046e columnFamilyName A 2024-11-26T15:27:42,360 DEBUG [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:42,361 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] regionserver.HStore(327): Store=ade03e3421a188de043c46d5d985046e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:27:42,361 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,362 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ade03e3421a188de043c46d5d985046e columnFamilyName B 2024-11-26T15:27:42,362 DEBUG [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:42,362 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] regionserver.HStore(327): Store=ade03e3421a188de043c46d5d985046e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:27:42,362 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,363 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ade03e3421a188de043c46d5d985046e columnFamilyName C 2024-11-26T15:27:42,363 DEBUG [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:42,363 INFO [StoreOpener-ade03e3421a188de043c46d5d985046e-1 {}] regionserver.HStore(327): Store=ade03e3421a188de043c46d5d985046e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:27:42,363 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,364 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,364 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,365 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,365 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,366 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T15:27:42,367 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,367 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened ade03e3421a188de043c46d5d985046e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61555631, jitterRate=-0.0827496200799942}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T15:27:42,367 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,368 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for ade03e3421a188de043c46d5d985046e: Running coprocessor pre-open hook at 1732634862356Writing region info on filesystem at 1732634862356Initializing all the Stores at 1732634862358 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634862358Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634862358Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634862358Cleaning up temporary data from old regions at 1732634862365 (+7 ms)Running coprocessor post-open hooks at 1732634862367 (+2 ms)Region opened successfully at 1732634862368 (+1 ms) 2024-11-26T15:27:42,369 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., pid=147, masterSystemTime=1732634862350 2024-11-26T15:27:42,370 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:42,370 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:42,370 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=ade03e3421a188de043c46d5d985046e, regionState=OPEN, openSeqNum=5, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:27:42,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-26T15:27:42,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 in 174 msec 2024-11-26T15:27:42,375 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-26T15:27:42,375 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ade03e3421a188de043c46d5d985046e, REOPEN/MOVE in 496 msec 2024-11-26T15:27:42,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-26T15:27:42,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 503 msec 2024-11-26T15:27:42,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 926 msec 2024-11-26T15:27:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-26T15:27:42,379 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:42,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d1887e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:42,380 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:42,381 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22bb5b24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:42,382 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:42,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302ce37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e2a1e31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:42,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:42,383 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:42,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:42,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a53d143, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:42,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:42,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:42,384 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44f71026, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdeed38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,385 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:42,385 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:42,386 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f356d73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:42,387 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@651460b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,388 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b6eabbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,388 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:42,388 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:42,388 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b320770, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:42,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:42,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:42,389 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:42,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a1d5f70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b3be2dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,390 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:42,390 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:42,391 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@177a9972, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:42,392 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:42,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3abc6070, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18d7368f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,392 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:42,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:42,393 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@513bea44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:42,394 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:42,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0cd8e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108b0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,395 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:42,395 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:42,395 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5201e3d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:42,396 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:42,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66d68bf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a5be7c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,397 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:42,397 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:42,398 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@758841a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:42,399 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bad4bd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@162f1ee9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,400 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:42,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:42,400 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:42,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:42,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:42,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f934543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:42,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:42,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:42,401 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:42,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e18b5b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e62e123, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,37081,-1 for getting cluster id 2024-11-26T15:27:42,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:27:42,403 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'abffd310-7d55-49f7-a58a-b264a94194b6' 2024-11-26T15:27:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:27:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "abffd310-7d55-49f7-a58a-b264a94194b6" 2024-11-26T15:27:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5101678e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,37081,-1] 2024-11-26T15:27:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:27:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:27:42,404 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:27:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a0c7e86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:27:42,405 DEBUG [Thread-2338 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:42,405 DEBUG [Thread-2357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:42,405 DEBUG [Thread-2342 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:42,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:27:42,405 DEBUG [Thread-2340 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:42,406 DEBUG [Thread-2349 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:42,406 DEBUG [Thread-2346 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:42,406 DEBUG [Thread-2344 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:42,407 DEBUG [Thread-2355 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:42,407 DEBUG [Thread-2351 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-26T15:27:42,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:42,408 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:42,408 DEBUG [Thread-2351 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:42,409 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:27:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T15:27:42,409 DEBUG [Thread-2351 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:42,409 DEBUG [Thread-2353 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:27:42,409 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:42,409 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:42,409 DEBUG [Thread-2355 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:42,410 DEBUG [Thread-2355 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:42,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:42,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:42,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:42,410 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:27:42,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:27:42,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,45729,1732634697146, seqNum=-1] 2024-11-26T15:27:42,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:27:42,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:27:42,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:42,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:42,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:27:42,412 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:27:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:27:42,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:27:42,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:27:42,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:27:42,414 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:27:42,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:27:42,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:27:42,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5] 2024-11-26T15:27:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,418 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:42,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ade03e3421a188de043c46d5d985046e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T15:27:42,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c6a634a41cd24741a3d95c9bce5680a0_ade03e3421a188de043c46d5d985046e is 50, key is test_row_0/A:col10/1732634862417/Put/seqid=0 2024-11-26T15:27:42,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634922438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634922438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:42,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634922438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:42,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634922442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634922442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:42,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742346_1522 (size=14594) 2024-11-26T15:27:42,453 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:42,456 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c6a634a41cd24741a3d95c9bce5680a0_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c6a634a41cd24741a3d95c9bce5680a0_ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:42,457 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/6b7aeb7120b346e1a3c8c699d247ac2e, store: [table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:42,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/6b7aeb7120b346e1a3c8c699d247ac2e is 175, key is test_row_0/A:col10/1732634862417/Put/seqid=0 2024-11-26T15:27:42,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742347_1523 (size=39549) 2024-11-26T15:27:42,470 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/6b7aeb7120b346e1a3c8c699d247ac2e 2024-11-26T15:27:42,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/9a38040a30f44ae4b898ab90b98b589d is 50, key is test_row_0/B:col10/1732634862417/Put/seqid=0 2024-11-26T15:27:42,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742348_1524 (size=12001) 2024-11-26T15:27:42,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T15:27:42,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634922552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:42,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634922553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634922553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634922553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634922553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:42,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T15:27:42,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:42,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:42,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:42,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:42,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:42,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T15:27:42,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:42,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:42,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:42,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:42,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T15:27:42,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634922763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:42,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634922763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:42,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634922763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:42,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634922763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:42,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634922763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:42,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T15:27:42,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:42,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:42,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:42,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:42,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:42,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/9a38040a30f44ae4b898ab90b98b589d 2024-11-26T15:27:42,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/5c68ac94ccfb4336aae28fa31867887f is 50, key is test_row_0/C:col10/1732634862417/Put/seqid=0 2024-11-26T15:27:42,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742349_1525 (size=12001) 2024-11-26T15:27:43,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T15:27:43,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:43,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:43,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:43,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:43,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T15:27:43,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634923072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634923072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634923072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634923072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634923073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T15:27:43,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:43,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:43,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:43,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:43,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:43,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/5c68ac94ccfb4336aae28fa31867887f 2024-11-26T15:27:43,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/6b7aeb7120b346e1a3c8c699d247ac2e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/6b7aeb7120b346e1a3c8c699d247ac2e 2024-11-26T15:27:43,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T15:27:43,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:43,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:43,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:43,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:43,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:43,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:43,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/6b7aeb7120b346e1a3c8c699d247ac2e, entries=200, sequenceid=15, filesize=38.6 K 2024-11-26T15:27:43,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/9a38040a30f44ae4b898ab90b98b589d as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/9a38040a30f44ae4b898ab90b98b589d 2024-11-26T15:27:43,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/9a38040a30f44ae4b898ab90b98b589d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-26T15:27:43,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/5c68ac94ccfb4336aae28fa31867887f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/5c68ac94ccfb4336aae28fa31867887f 2024-11-26T15:27:43,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/5c68ac94ccfb4336aae28fa31867887f, entries=150, sequenceid=15, filesize=11.7 K 2024-11-26T15:27:43,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ade03e3421a188de043c46d5d985046e in 919ms, sequenceid=15, compaction requested=false 2024-11-26T15:27:43,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:43,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T15:27:43,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:43,479 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing ade03e3421a188de043c46d5d985046e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-26T15:27:43,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264de566f68ce84c628a98e1978c48f152_ade03e3421a188de043c46d5d985046e is 50, key is test_row_0/A:col10/1732634862437/Put/seqid=0 2024-11-26T15:27:43,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742350_1526 (size=12154) 2024-11-26T15:27:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T15:27:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:43,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:43,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634923589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634923590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634923592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634923593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634923593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634923703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634923703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634923703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634923703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634923703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:43,893 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264de566f68ce84c628a98e1978c48f152_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264de566f68ce84c628a98e1978c48f152_ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:43,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/9873a12ff9dc4709a025ce79a0d2069d, store: [table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:43,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/9873a12ff9dc4709a025ce79a0d2069d is 175, key is test_row_0/A:col10/1732634862437/Put/seqid=0 2024-11-26T15:27:43,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742351_1527 (size=30955) 2024-11-26T15:27:43,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634923913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634923913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634923913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634923913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:43,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634923913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:43,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:43,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:44,106 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T15:27:44,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634924222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:44,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634924222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634924223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634924223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634924223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:44,297 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/9873a12ff9dc4709a025ce79a0d2069d 2024-11-26T15:27:44,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/9f45ff24c47342cda0b07fdcf10730f4 is 50, key is test_row_0/B:col10/1732634862437/Put/seqid=0 2024-11-26T15:27:44,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742352_1528 (size=12001) 2024-11-26T15:27:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T15:27:44,706 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/9f45ff24c47342cda0b07fdcf10730f4 2024-11-26T15:27:44,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/c7a77910bc59469f9bb0a01b4b2369d9 is 50, key is test_row_0/C:col10/1732634862437/Put/seqid=0 2024-11-26T15:27:44,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742353_1529 (size=12001) 2024-11-26T15:27:44,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:44,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634924732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:44,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634924733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634924733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:44,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634924733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:44,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634924733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:45,115 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/c7a77910bc59469f9bb0a01b4b2369d9 2024-11-26T15:27:45,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/9873a12ff9dc4709a025ce79a0d2069d as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9873a12ff9dc4709a025ce79a0d2069d 2024-11-26T15:27:45,121 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9873a12ff9dc4709a025ce79a0d2069d, entries=150, sequenceid=40, filesize=30.2 K 2024-11-26T15:27:45,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/9f45ff24c47342cda0b07fdcf10730f4 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/9f45ff24c47342cda0b07fdcf10730f4 2024-11-26T15:27:45,124 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/9f45ff24c47342cda0b07fdcf10730f4, entries=150, sequenceid=40, filesize=11.7 K 2024-11-26T15:27:45,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/c7a77910bc59469f9bb0a01b4b2369d9 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/c7a77910bc59469f9bb0a01b4b2369d9 2024-11-26T15:27:45,127 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/c7a77910bc59469f9bb0a01b4b2369d9, entries=150, sequenceid=40, filesize=11.7 K 2024-11-26T15:27:45,128 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for ade03e3421a188de043c46d5d985046e in 1650ms, sequenceid=40, compaction requested=false 2024-11-26T15:27:45,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:45,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:45,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-26T15:27:45,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-26T15:27:45,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-26T15:27:45,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7190 sec 2024-11-26T15:27:45,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.7260 sec 2024-11-26T15:27:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:45,744 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:45,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ade03e3421a188de043c46d5d985046e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-26T15:27:45,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c32dff3caacb4484a13281e1f5275d8e_ade03e3421a188de043c46d5d985046e is 50, key is test_row_0/A:col10/1732634863591/Put/seqid=0 2024-11-26T15:27:45,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742354_1530 (size=14594) 2024-11-26T15:27:45,757 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:45,760 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c32dff3caacb4484a13281e1f5275d8e_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c32dff3caacb4484a13281e1f5275d8e_ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:45,760 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/9da57d4d15054f5b86bcdaf925674b62, store: [table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:45,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/9da57d4d15054f5b86bcdaf925674b62 is 175, key is test_row_0/A:col10/1732634863591/Put/seqid=0 2024-11-26T15:27:45,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634925766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634925767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:45,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742355_1531 (size=39549) 2024-11-26T15:27:45,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634925769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634925770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:45,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634925772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:45,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634925872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:45,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634925882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634925882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:45,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634925882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634925882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:46,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634926082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:46,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634926092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:46,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634926092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634926092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634926093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:46,174 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/9da57d4d15054f5b86bcdaf925674b62 2024-11-26T15:27:46,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/8d76f18527ef4a0d9cfbc621f9998732 is 50, key is test_row_0/B:col10/1732634863591/Put/seqid=0 2024-11-26T15:27:46,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742356_1532 (size=12001) 2024-11-26T15:27:46,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634926393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:46,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634926402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:46,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634926402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:46,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634926402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:46,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634926402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T15:27:46,573 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:46,573 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:27:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-26T15:27:46,575 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:27:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T15:27:46,575 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:27:46,575 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:27:46,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/8d76f18527ef4a0d9cfbc621f9998732 2024-11-26T15:27:46,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/017e32b4f99440a291eda6c3146241a5 is 50, key is test_row_0/C:col10/1732634863591/Put/seqid=0 2024-11-26T15:27:46,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742357_1533 (size=12001) 2024-11-26T15:27:46,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/017e32b4f99440a291eda6c3146241a5 2024-11-26T15:27:46,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/9da57d4d15054f5b86bcdaf925674b62 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9da57d4d15054f5b86bcdaf925674b62 2024-11-26T15:27:46,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9da57d4d15054f5b86bcdaf925674b62, entries=200, sequenceid=54, filesize=38.6 K 2024-11-26T15:27:46,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/8d76f18527ef4a0d9cfbc621f9998732 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/8d76f18527ef4a0d9cfbc621f9998732 2024-11-26T15:27:46,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/8d76f18527ef4a0d9cfbc621f9998732, entries=150, sequenceid=54, filesize=11.7 K 2024-11-26T15:27:46,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/017e32b4f99440a291eda6c3146241a5 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/017e32b4f99440a291eda6c3146241a5 2024-11-26T15:27:46,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/017e32b4f99440a291eda6c3146241a5, entries=150, sequenceid=54, filesize=11.7 K 2024-11-26T15:27:46,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for ade03e3421a188de043c46d5d985046e in 872ms, sequenceid=54, compaction requested=true 2024-11-26T15:27:46,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:46,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:27:46,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:46,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:27:46,617 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:46,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:46,617 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:46,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:27:46,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:46,617 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:46,617 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:46,617 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/B is initiating minor compaction (all files) 2024-11-26T15:27:46,617 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/A is initiating minor compaction (all files) 2024-11-26T15:27:46,617 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/B in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:46,617 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/A in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:46,617 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/9a38040a30f44ae4b898ab90b98b589d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/9f45ff24c47342cda0b07fdcf10730f4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/8d76f18527ef4a0d9cfbc621f9998732] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=35.2 K 2024-11-26T15:27:46,617 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/6b7aeb7120b346e1a3c8c699d247ac2e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9873a12ff9dc4709a025ce79a0d2069d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9da57d4d15054f5b86bcdaf925674b62] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=107.5 K 2024-11-26T15:27:46,617 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:46,617 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/6b7aeb7120b346e1a3c8c699d247ac2e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9873a12ff9dc4709a025ce79a0d2069d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9da57d4d15054f5b86bcdaf925674b62] 2024-11-26T15:27:46,618 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a38040a30f44ae4b898ab90b98b589d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732634862413 2024-11-26T15:27:46,618 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b7aeb7120b346e1a3c8c699d247ac2e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732634862413 2024-11-26T15:27:46,618 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f45ff24c47342cda0b07fdcf10730f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732634862437 2024-11-26T15:27:46,618 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9873a12ff9dc4709a025ce79a0d2069d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732634862437 2024-11-26T15:27:46,618 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9da57d4d15054f5b86bcdaf925674b62, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732634863588 2024-11-26T15:27:46,618 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d76f18527ef4a0d9cfbc621f9998732, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732634863588 2024-11-26T15:27:46,623 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:46,624 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#B#compaction#452 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:46,624 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411266cbc48a563e04ddb82299090db7c185e_ade03e3421a188de043c46d5d985046e store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:46,624 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/b22d82be3c3b429e96f8ee1d272a2b39 is 50, key is test_row_0/B:col10/1732634863591/Put/seqid=0 2024-11-26T15:27:46,626 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411266cbc48a563e04ddb82299090db7c185e_ade03e3421a188de043c46d5d985046e, store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:46,626 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266cbc48a563e04ddb82299090db7c185e_ade03e3421a188de043c46d5d985046e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:46,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742358_1534 (size=12104) 2024-11-26T15:27:46,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742359_1535 (size=4469) 2024-11-26T15:27:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T15:27:46,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-26T15:27:46,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:46,727 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing ade03e3421a188de043c46d5d985046e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-26T15:27:46,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cee64ffd523f4f60a9de9f7b26293a23_ade03e3421a188de043c46d5d985046e is 50, key is test_row_0/A:col10/1732634865769/Put/seqid=0 2024-11-26T15:27:46,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742360_1536 (size=12154) 2024-11-26T15:27:46,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:46,740 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cee64ffd523f4f60a9de9f7b26293a23_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cee64ffd523f4f60a9de9f7b26293a23_ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:46,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/4605b4e8a15748039a3634308d462954, store: [table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:46,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/4605b4e8a15748039a3634308d462954 is 175, key is test_row_0/A:col10/1732634865769/Put/seqid=0 2024-11-26T15:27:46,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742361_1537 (size=30955) 2024-11-26T15:27:46,745 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/4605b4e8a15748039a3634308d462954 2024-11-26T15:27:46,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/bdce483fcf97457699cc4f08dcb4fe2b is 50, key is test_row_0/B:col10/1732634865769/Put/seqid=0 2024-11-26T15:27:46,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742362_1538 (size=12001) 2024-11-26T15:27:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T15:27:46,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:46,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634926917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634926917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:46,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:46,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634926918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:46,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:46,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634926920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634926921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:47,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:47,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634927023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:47,031 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/b22d82be3c3b429e96f8ee1d272a2b39 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/b22d82be3c3b429e96f8ee1d272a2b39 2024-11-26T15:27:47,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634927032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:47,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634927032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:47,035 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ade03e3421a188de043c46d5d985046e/B of ade03e3421a188de043c46d5d985046e into b22d82be3c3b429e96f8ee1d272a2b39(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:47,035 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:47,035 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/B, priority=13, startTime=1732634866616; duration=0sec 2024-11-26T15:27:47,035 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:47,035 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:B 2024-11-26T15:27:47,035 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:47,035 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#A#compaction#453 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:47,035 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:47,036 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/C is initiating minor compaction (all files) 2024-11-26T15:27:47,036 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/C in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:47,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:47,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:47,036 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/5c68ac94ccfb4336aae28fa31867887f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/c7a77910bc59469f9bb0a01b4b2369d9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/017e32b4f99440a291eda6c3146241a5] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=35.2 K 2024-11-26T15:27:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634927032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634927032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:47,036 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/dbe05ded48494db685a7d729dc99fc79 is 175, key is test_row_0/A:col10/1732634863591/Put/seqid=0 2024-11-26T15:27:47,036 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c68ac94ccfb4336aae28fa31867887f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732634862413 2024-11-26T15:27:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:47,037 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting c7a77910bc59469f9bb0a01b4b2369d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732634862437 2024-11-26T15:27:47,037 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 017e32b4f99440a291eda6c3146241a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732634863588 2024-11-26T15:27:47,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742363_1539 (size=31058) 2024-11-26T15:27:47,041 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/dbe05ded48494db685a7d729dc99fc79 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/dbe05ded48494db685a7d729dc99fc79 2024-11-26T15:27:47,042 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#C#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:47,043 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/3007539562524785b4ceccb75fe80caf is 50, key is test_row_0/C:col10/1732634863591/Put/seqid=0 2024-11-26T15:27:47,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742364_1540 (size=12104) 2024-11-26T15:27:47,047 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ade03e3421a188de043c46d5d985046e/A of ade03e3421a188de043c46d5d985046e into dbe05ded48494db685a7d729dc99fc79(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:47,047 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:47,047 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/A, priority=13, startTime=1732634866616; duration=0sec 2024-11-26T15:27:47,047 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:47,048 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:A 2024-11-26T15:27:47,156 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/bdce483fcf97457699cc4f08dcb4fe2b 2024-11-26T15:27:47,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/1a21b727c7cf4dfc89a6746c25a51043 is 50, key is test_row_0/C:col10/1732634865769/Put/seqid=0 2024-11-26T15:27:47,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742365_1541 (size=12001) 2024-11-26T15:27:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T15:27:47,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634927233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:47,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634927242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634927242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:47,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634927242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:47,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634927242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:47,451 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/3007539562524785b4ceccb75fe80caf as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/3007539562524785b4ceccb75fe80caf 2024-11-26T15:27:47,454 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ade03e3421a188de043c46d5d985046e/C of ade03e3421a188de043c46d5d985046e into 3007539562524785b4ceccb75fe80caf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:47,454 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:47,454 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/C, priority=13, startTime=1732634866617; duration=0sec 2024-11-26T15:27:47,454 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:47,454 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:C 2024-11-26T15:27:47,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634927542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:47,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:47,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634927552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:47,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634927553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634927553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:47,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634927553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:47,564 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/1a21b727c7cf4dfc89a6746c25a51043 2024-11-26T15:27:47,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/4605b4e8a15748039a3634308d462954 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/4605b4e8a15748039a3634308d462954 2024-11-26T15:27:47,570 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/4605b4e8a15748039a3634308d462954, entries=150, sequenceid=76, filesize=30.2 K 2024-11-26T15:27:47,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/bdce483fcf97457699cc4f08dcb4fe2b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/bdce483fcf97457699cc4f08dcb4fe2b 2024-11-26T15:27:47,574 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/bdce483fcf97457699cc4f08dcb4fe2b, entries=150, sequenceid=76, filesize=11.7 K 2024-11-26T15:27:47,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/1a21b727c7cf4dfc89a6746c25a51043 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/1a21b727c7cf4dfc89a6746c25a51043 2024-11-26T15:27:47,578 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/1a21b727c7cf4dfc89a6746c25a51043, entries=150, sequenceid=76, filesize=11.7 K 2024-11-26T15:27:47,578 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ade03e3421a188de043c46d5d985046e in 851ms, sequenceid=76, compaction requested=false 2024-11-26T15:27:47,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:47,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:47,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-26T15:27:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-26T15:27:47,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-26T15:27:47,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0050 sec 2024-11-26T15:27:47,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.0090 sec 2024-11-26T15:27:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T15:27:47,712 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:47,713 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:27:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-26T15:27:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T15:27:47,755 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:27:47,756 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:27:47,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:27:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T15:27:47,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-26T15:27:47,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:47,908 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing ade03e3421a188de043c46d5d985046e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-26T15:27:47,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261cdc74972acd469b9135492a13fa3142_ade03e3421a188de043c46d5d985046e is 50, key is test_row_0/A:col10/1732634866915/Put/seqid=0 2024-11-26T15:27:47,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742366_1542 (size=12154) 2024-11-26T15:27:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T15:27:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:48,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:48,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634928076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634928077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634928078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634928078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634928079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634928182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634928192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634928192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634928193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634928193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:48,319 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261cdc74972acd469b9135492a13fa3142_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261cdc74972acd469b9135492a13fa3142_ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:48,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/7fcdebd28bc8421e8f4ae3a482162c51, store: [table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:48,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/7fcdebd28bc8421e8f4ae3a482162c51 is 175, key is test_row_0/A:col10/1732634866915/Put/seqid=0 2024-11-26T15:27:48,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742367_1543 (size=30955) 2024-11-26T15:27:48,324 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/7fcdebd28bc8421e8f4ae3a482162c51 2024-11-26T15:27:48,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/80124ca6db5640148f14a05217768ed7 is 50, key is test_row_0/B:col10/1732634866915/Put/seqid=0 2024-11-26T15:27:48,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742368_1544 (size=12001) 2024-11-26T15:27:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T15:27:48,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634928392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634928402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634928403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634928403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634928403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634928702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634928712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634928712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634928712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634928713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:48,733 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/80124ca6db5640148f14a05217768ed7 2024-11-26T15:27:48,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/bad99cef66524af0bf97464ed84c063d is 50, key is test_row_0/C:col10/1732634866915/Put/seqid=0 2024-11-26T15:27:48,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742369_1545 (size=12001) 2024-11-26T15:27:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T15:27:49,141 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/bad99cef66524af0bf97464ed84c063d 2024-11-26T15:27:49,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/7fcdebd28bc8421e8f4ae3a482162c51 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/7fcdebd28bc8421e8f4ae3a482162c51 2024-11-26T15:27:49,147 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/7fcdebd28bc8421e8f4ae3a482162c51, entries=150, sequenceid=93, filesize=30.2 K 2024-11-26T15:27:49,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/80124ca6db5640148f14a05217768ed7 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/80124ca6db5640148f14a05217768ed7 2024-11-26T15:27:49,150 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/80124ca6db5640148f14a05217768ed7, entries=150, sequenceid=93, filesize=11.7 K 2024-11-26T15:27:49,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/bad99cef66524af0bf97464ed84c063d as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/bad99cef66524af0bf97464ed84c063d 2024-11-26T15:27:49,154 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/bad99cef66524af0bf97464ed84c063d, entries=150, sequenceid=93, filesize=11.7 K 2024-11-26T15:27:49,158 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for ade03e3421a188de043c46d5d985046e in 1250ms, sequenceid=93, compaction requested=true 2024-11-26T15:27:49,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:49,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:49,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-26T15:27:49,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-26T15:27:49,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-26T15:27:49,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4030 sec 2024-11-26T15:27:49,162 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.4480 sec 2024-11-26T15:27:49,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:49,215 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:49,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ade03e3421a188de043c46d5d985046e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-26T15:27:49,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126594df66c086b4874bd2c99c495e3a6a0_ade03e3421a188de043c46d5d985046e is 50, key is test_row_0/A:col10/1732634868072/Put/seqid=0 2024-11-26T15:27:49,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634929228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634929228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742370_1546 (size=14594) 2024-11-26T15:27:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,234 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:49,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634929231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,237 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126594df66c086b4874bd2c99c495e3a6a0_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126594df66c086b4874bd2c99c495e3a6a0_ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:49,238 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/850f3bab17104b678c6fdb7dca8fc9bb, store: [table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:49,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634929232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/850f3bab17104b678c6fdb7dca8fc9bb is 175, key is test_row_0/A:col10/1732634868072/Put/seqid=0 2024-11-26T15:27:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634929233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742371_1547 (size=39549) 2024-11-26T15:27:49,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634929342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634929342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634929342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634929342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634929342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634929552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634929552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634929552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634929552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634929552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,642 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/850f3bab17104b678c6fdb7dca8fc9bb 2024-11-26T15:27:49,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/0eb15181927c4bfc9d32b4c2c86202ab is 50, key is test_row_0/B:col10/1732634868072/Put/seqid=0 2024-11-26T15:27:49,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742372_1548 (size=12001) 2024-11-26T15:27:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T15:27:49,863 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:49,863 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:27:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-26T15:27:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T15:27:49,864 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:27:49,865 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:27:49,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:27:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634929863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634929863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634929863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634929863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634929863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T15:27:50,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T15:27:50,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:50,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:50,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:50,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:50,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:50,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/0eb15181927c4bfc9d32b4c2c86202ab 2024-11-26T15:27:50,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/e9fea665a9494cb0baeddb17bb0a6d0f is 50, key is test_row_0/C:col10/1732634868072/Put/seqid=0 2024-11-26T15:27:50,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742373_1549 (size=12001) 2024-11-26T15:27:50,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T15:27:50,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:50,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:50,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:50,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:50,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:50,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T15:27:50,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T15:27:50,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:50,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:50,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:50,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:50,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:50,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:50,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:50,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634930372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:50,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:50,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634930372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:50,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:50,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634930372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:50,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:50,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634930372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:50,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:50,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634930372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:50,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/e9fea665a9494cb0baeddb17bb0a6d0f 2024-11-26T15:27:50,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/850f3bab17104b678c6fdb7dca8fc9bb as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/850f3bab17104b678c6fdb7dca8fc9bb 2024-11-26T15:27:50,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/850f3bab17104b678c6fdb7dca8fc9bb, entries=200, sequenceid=116, filesize=38.6 K 2024-11-26T15:27:50,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/0eb15181927c4bfc9d32b4c2c86202ab as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0eb15181927c4bfc9d32b4c2c86202ab 2024-11-26T15:27:50,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0eb15181927c4bfc9d32b4c2c86202ab, entries=150, sequenceid=116, filesize=11.7 K 2024-11-26T15:27:50,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/e9fea665a9494cb0baeddb17bb0a6d0f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e9fea665a9494cb0baeddb17bb0a6d0f 2024-11-26T15:27:50,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e9fea665a9494cb0baeddb17bb0a6d0f, entries=150, sequenceid=116, filesize=11.7 K 2024-11-26T15:27:50,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for ade03e3421a188de043c46d5d985046e in 1258ms, sequenceid=116, compaction requested=true 2024-11-26T15:27:50,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:50,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:27:50,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:50,473 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:27:50,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:27:50,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:50,473 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:27:50,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:27:50,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:50,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T15:27:50,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:50,474 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing ade03e3421a188de043c46d5d985046e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-26T15:27:50,474 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:27:50,474 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/A is initiating minor compaction (all files) 2024-11-26T15:27:50,474 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/A in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:50,474 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/dbe05ded48494db685a7d729dc99fc79, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/4605b4e8a15748039a3634308d462954, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/7fcdebd28bc8421e8f4ae3a482162c51, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/850f3bab17104b678c6fdb7dca8fc9bb] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=129.4 K 2024-11-26T15:27:50,474 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:50,474 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/dbe05ded48494db685a7d729dc99fc79, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/4605b4e8a15748039a3634308d462954, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/7fcdebd28bc8421e8f4ae3a482162c51, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/850f3bab17104b678c6fdb7dca8fc9bb] 2024-11-26T15:27:50,474 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:27:50,474 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/B is initiating minor compaction (all files) 2024-11-26T15:27:50,474 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/B in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:50,475 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/b22d82be3c3b429e96f8ee1d272a2b39, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/bdce483fcf97457699cc4f08dcb4fe2b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/80124ca6db5640148f14a05217768ed7, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0eb15181927c4bfc9d32b4c2c86202ab] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=47.0 K 2024-11-26T15:27:50,475 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbe05ded48494db685a7d729dc99fc79, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732634863588 2024-11-26T15:27:50,475 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting b22d82be3c3b429e96f8ee1d272a2b39, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732634863588 2024-11-26T15:27:50,475 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting bdce483fcf97457699cc4f08dcb4fe2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732634865763 2024-11-26T15:27:50,475 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4605b4e8a15748039a3634308d462954, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732634865763 2024-11-26T15:27:50,475 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fcdebd28bc8421e8f4ae3a482162c51, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732634866915 2024-11-26T15:27:50,475 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 80124ca6db5640148f14a05217768ed7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732634866915 2024-11-26T15:27:50,476 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 850f3bab17104b678c6fdb7dca8fc9bb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732634868072 2024-11-26T15:27:50,476 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eb15181927c4bfc9d32b4c2c86202ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732634868072 2024-11-26T15:27:50,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112643b56d7b5b614483b8b100e15748fba8_ade03e3421a188de043c46d5d985046e is 50, key is test_row_0/A:col10/1732634869232/Put/seqid=0 2024-11-26T15:27:50,481 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:50,483 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126c7f3f132f0444ca890d2355ec8cac94a_ade03e3421a188de043c46d5d985046e store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:50,484 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#B#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:50,484 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/aec443b8542a47e3b4860078b50a059c is 50, key is test_row_0/B:col10/1732634868072/Put/seqid=0 2024-11-26T15:27:50,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742374_1550 (size=12154) 2024-11-26T15:27:50,485 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126c7f3f132f0444ca890d2355ec8cac94a_ade03e3421a188de043c46d5d985046e, store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:50,485 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c7f3f132f0444ca890d2355ec8cac94a_ade03e3421a188de043c46d5d985046e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:50,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742375_1551 (size=12241) 2024-11-26T15:27:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T15:27:50,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742376_1552 (size=4469) 2024-11-26T15:27:50,494 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#A#compaction#465 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:50,495 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/9a8dded33d934c98a1c5e24acf020c4f is 175, key is test_row_0/A:col10/1732634868072/Put/seqid=0 2024-11-26T15:27:50,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742377_1553 (size=31195) 2024-11-26T15:27:50,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:50,887 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112643b56d7b5b614483b8b100e15748fba8_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112643b56d7b5b614483b8b100e15748fba8_ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:50,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/3312b339ba5545658ea6c43c462c3dc3, store: [table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:50,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/3312b339ba5545658ea6c43c462c3dc3 is 175, key is test_row_0/A:col10/1732634869232/Put/seqid=0 2024-11-26T15:27:50,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742378_1554 (size=30955) 2024-11-26T15:27:50,893 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/aec443b8542a47e3b4860078b50a059c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/aec443b8542a47e3b4860078b50a059c 2024-11-26T15:27:50,896 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ade03e3421a188de043c46d5d985046e/B of ade03e3421a188de043c46d5d985046e into aec443b8542a47e3b4860078b50a059c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:50,896 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:50,896 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/B, priority=12, startTime=1732634870473; duration=0sec 2024-11-26T15:27:50,896 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:50,896 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:B 2024-11-26T15:27:50,896 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:27:50,897 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T15:27:50,897 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/C is initiating minor compaction (all files) 2024-11-26T15:27:50,897 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/C in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:50,897 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/3007539562524785b4ceccb75fe80caf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/1a21b727c7cf4dfc89a6746c25a51043, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/bad99cef66524af0bf97464ed84c063d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e9fea665a9494cb0baeddb17bb0a6d0f] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=47.0 K 2024-11-26T15:27:50,898 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 3007539562524785b4ceccb75fe80caf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732634863588 2024-11-26T15:27:50,898 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a21b727c7cf4dfc89a6746c25a51043, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732634865763 2024-11-26T15:27:50,898 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting bad99cef66524af0bf97464ed84c063d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732634866915 2024-11-26T15:27:50,898 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting e9fea665a9494cb0baeddb17bb0a6d0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732634868072 2024-11-26T15:27:50,901 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/9a8dded33d934c98a1c5e24acf020c4f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9a8dded33d934c98a1c5e24acf020c4f 2024-11-26T15:27:50,904 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ade03e3421a188de043c46d5d985046e/A of ade03e3421a188de043c46d5d985046e into 9a8dded33d934c98a1c5e24acf020c4f(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:50,904 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:50,904 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/A, priority=12, startTime=1732634870473; duration=0sec 2024-11-26T15:27:50,904 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#C#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:50,904 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:50,904 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:A 2024-11-26T15:27:50,905 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/bd9a33b2c07b48f789c5f356446f7ea2 is 50, key is test_row_0/C:col10/1732634868072/Put/seqid=0 2024-11-26T15:27:50,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742379_1555 (size=12241) 2024-11-26T15:27:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T15:27:51,291 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/3312b339ba5545658ea6c43c462c3dc3 2024-11-26T15:27:51,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/0e583294dc024c6e86724614e6d3d71c is 50, key is test_row_0/B:col10/1732634869232/Put/seqid=0 2024-11-26T15:27:51,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742380_1556 (size=12001) 2024-11-26T15:27:51,311 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/bd9a33b2c07b48f789c5f356446f7ea2 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/bd9a33b2c07b48f789c5f356446f7ea2 2024-11-26T15:27:51,314 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ade03e3421a188de043c46d5d985046e/C of ade03e3421a188de043c46d5d985046e into bd9a33b2c07b48f789c5f356446f7ea2(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:51,314 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:51,314 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/C, priority=12, startTime=1732634870473; duration=0sec 2024-11-26T15:27:51,314 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:51,315 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:C 2024-11-26T15:27:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:51,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:51,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634931402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:51,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634931404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:51,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634931405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:51,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634931407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:51,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:51,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634931408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:51,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:51,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634931512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:51,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634931512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:51,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634931512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:51,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634931523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:51,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634931523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:51,699 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/0e583294dc024c6e86724614e6d3d71c 2024-11-26T15:27:51,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/3e47c8fb85584815aca05da73cba3034 is 50, key is test_row_0/C:col10/1732634869232/Put/seqid=0 2024-11-26T15:27:51,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742381_1557 (size=12001) 2024-11-26T15:27:51,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634931723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634931723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634931723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:51,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634931732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:51,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634931732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T15:27:52,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634932033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634932033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634932033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634932042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634932042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,108 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/3e47c8fb85584815aca05da73cba3034 2024-11-26T15:27:52,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/3312b339ba5545658ea6c43c462c3dc3 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/3312b339ba5545658ea6c43c462c3dc3 2024-11-26T15:27:52,114 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/3312b339ba5545658ea6c43c462c3dc3, entries=150, sequenceid=129, filesize=30.2 K 2024-11-26T15:27:52,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/0e583294dc024c6e86724614e6d3d71c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0e583294dc024c6e86724614e6d3d71c 2024-11-26T15:27:52,117 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0e583294dc024c6e86724614e6d3d71c, entries=150, sequenceid=129, filesize=11.7 K 2024-11-26T15:27:52,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/3e47c8fb85584815aca05da73cba3034 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/3e47c8fb85584815aca05da73cba3034 2024-11-26T15:27:52,120 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/3e47c8fb85584815aca05da73cba3034, entries=150, sequenceid=129, filesize=11.7 K 2024-11-26T15:27:52,120 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ade03e3421a188de043c46d5d985046e in 1646ms, sequenceid=129, compaction requested=false 2024-11-26T15:27:52,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:52,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:52,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-26T15:27:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-26T15:27:52,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-26T15:27:52,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2570 sec 2024-11-26T15:27:52,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.2600 sec 2024-11-26T15:27:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:52,545 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:52,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ade03e3421a188de043c46d5d985046e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-26T15:27:52,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126536cd76fd8224094b92d1b0c725acf07_ade03e3421a188de043c46d5d985046e is 50, key is test_row_0/A:col10/1732634871401/Put/seqid=0 2024-11-26T15:27:52,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742382_1558 (size=12304) 2024-11-26T15:27:52,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634932552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634932553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634932554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634932556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634932557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,561 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634932662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634932662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634932662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634932662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634932872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634932872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634932872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634932872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:52,954 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:52,957 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126536cd76fd8224094b92d1b0c725acf07_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126536cd76fd8224094b92d1b0c725acf07_ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:52,957 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/37cd1b3c069040e1825f1a5034941c02, store: [table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:52,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/37cd1b3c069040e1825f1a5034941c02 is 175, key is test_row_0/A:col10/1732634871401/Put/seqid=0 2024-11-26T15:27:52,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742383_1559 (size=31105) 2024-11-26T15:27:53,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:53,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:53,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634933183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:53,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634933183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:53,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634933183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:53,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634933183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:53,361 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/37cd1b3c069040e1825f1a5034941c02 2024-11-26T15:27:53,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/cb602e0b99de46bb910ada95ba3d9b1b is 50, key is test_row_0/B:col10/1732634871401/Put/seqid=0 2024-11-26T15:27:53,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742384_1560 (size=12151) 2024-11-26T15:27:53,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634933563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:53,565 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in ade03e3421a188de043c46d5d985046e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:53,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:53,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634933693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:53,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:53,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634933693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:53,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634933693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:53,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634933693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:53,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/cb602e0b99de46bb910ada95ba3d9b1b 2024-11-26T15:27:53,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/c557f72a12854a238eefcbaa12c2dec4 is 50, key is test_row_0/C:col10/1732634871401/Put/seqid=0 2024-11-26T15:27:53,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742385_1561 (size=12151) 2024-11-26T15:27:54,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T15:27:54,033 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:54,033 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:27:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-26T15:27:54,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T15:27:54,035 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:27:54,035 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:27:54,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:27:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T15:27:54,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/c557f72a12854a238eefcbaa12c2dec4 2024-11-26T15:27:54,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/37cd1b3c069040e1825f1a5034941c02 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/37cd1b3c069040e1825f1a5034941c02 2024-11-26T15:27:54,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/37cd1b3c069040e1825f1a5034941c02, entries=150, sequenceid=156, filesize=30.4 K 2024-11-26T15:27:54,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/cb602e0b99de46bb910ada95ba3d9b1b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/cb602e0b99de46bb910ada95ba3d9b1b 2024-11-26T15:27:54,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T15:27:54,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:54,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:54,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:54,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:54,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:54,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/cb602e0b99de46bb910ada95ba3d9b1b, entries=150, sequenceid=156, filesize=11.9 K 2024-11-26T15:27:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:54,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/c557f72a12854a238eefcbaa12c2dec4 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/c557f72a12854a238eefcbaa12c2dec4 2024-11-26T15:27:54,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/c557f72a12854a238eefcbaa12c2dec4, entries=150, sequenceid=156, filesize=11.9 K 2024-11-26T15:27:54,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for ade03e3421a188de043c46d5d985046e in 1647ms, sequenceid=156, compaction requested=true 2024-11-26T15:27:54,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:54,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:27:54,192 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:54,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:54,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:27:54,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:54,192 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:54,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:27:54,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:54,193 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:54,193 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/B is initiating minor compaction (all files) 2024-11-26T15:27:54,194 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/B in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:54,194 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/aec443b8542a47e3b4860078b50a059c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0e583294dc024c6e86724614e6d3d71c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/cb602e0b99de46bb910ada95ba3d9b1b] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=35.5 K 2024-11-26T15:27:54,194 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93255 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:54,194 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/A is initiating minor compaction (all files) 2024-11-26T15:27:54,194 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/A in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:54,194 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9a8dded33d934c98a1c5e24acf020c4f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/3312b339ba5545658ea6c43c462c3dc3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/37cd1b3c069040e1825f1a5034941c02] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=91.1 K 2024-11-26T15:27:54,194 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:54,194 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9a8dded33d934c98a1c5e24acf020c4f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/3312b339ba5545658ea6c43c462c3dc3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/37cd1b3c069040e1825f1a5034941c02] 2024-11-26T15:27:54,194 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting aec443b8542a47e3b4860078b50a059c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732634868072 2024-11-26T15:27:54,194 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a8dded33d934c98a1c5e24acf020c4f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732634868072 2024-11-26T15:27:54,194 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e583294dc024c6e86724614e6d3d71c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732634869225 2024-11-26T15:27:54,195 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3312b339ba5545658ea6c43c462c3dc3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732634869225 2024-11-26T15:27:54,195 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting cb602e0b99de46bb910ada95ba3d9b1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732634871401 2024-11-26T15:27:54,195 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37cd1b3c069040e1825f1a5034941c02, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732634871401 2024-11-26T15:27:54,200 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#B#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:54,201 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/ea563d0af4294587a7f3d6e1180298ce is 50, key is test_row_0/B:col10/1732634871401/Put/seqid=0 2024-11-26T15:27:54,201 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:54,202 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411261fbadef9ffc6489584d13c5c47cad476_ade03e3421a188de043c46d5d985046e store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:54,204 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411261fbadef9ffc6489584d13c5c47cad476_ade03e3421a188de043c46d5d985046e, store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:54,204 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261fbadef9ffc6489584d13c5c47cad476_ade03e3421a188de043c46d5d985046e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:54,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742386_1562 (size=12493) 2024-11-26T15:27:54,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742387_1563 (size=4469) 2024-11-26T15:27:54,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T15:27:54,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:54,340 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing ade03e3421a188de043c46d5d985046e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-26T15:27:54,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126debcf9d848f14b57b37d512245d9f510_ade03e3421a188de043c46d5d985046e is 50, key is test_row_0/A:col10/1732634872555/Put/seqid=0 2024-11-26T15:27:54,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742388_1564 (size=12304) 2024-11-26T15:27:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T15:27:54,609 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#A#compaction#474 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:54,609 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/ea563d0af4294587a7f3d6e1180298ce as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/ea563d0af4294587a7f3d6e1180298ce 2024-11-26T15:27:54,609 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/984e150b47314eb3a254949277534611 is 175, key is test_row_0/A:col10/1732634871401/Put/seqid=0 2024-11-26T15:27:54,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742389_1565 (size=31447) 2024-11-26T15:27:54,612 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ade03e3421a188de043c46d5d985046e/B of ade03e3421a188de043c46d5d985046e into ea563d0af4294587a7f3d6e1180298ce(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:54,613 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:54,613 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/B, priority=13, startTime=1732634874192; duration=0sec 2024-11-26T15:27:54,613 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:54,613 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:B 2024-11-26T15:27:54,613 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:54,613 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:54,614 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/C is initiating minor compaction (all files) 2024-11-26T15:27:54,614 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/C in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:54,614 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/bd9a33b2c07b48f789c5f356446f7ea2, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/3e47c8fb85584815aca05da73cba3034, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/c557f72a12854a238eefcbaa12c2dec4] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=35.5 K 2024-11-26T15:27:54,614 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting bd9a33b2c07b48f789c5f356446f7ea2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732634868072 2024-11-26T15:27:54,614 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e47c8fb85584815aca05da73cba3034, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732634869225 2024-11-26T15:27:54,614 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting c557f72a12854a238eefcbaa12c2dec4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732634871401 2024-11-26T15:27:54,620 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#C#compaction#476 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:54,621 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/37546ae74735493fb7b19b8283bcefde is 50, key is test_row_0/C:col10/1732634871401/Put/seqid=0 2024-11-26T15:27:54,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742390_1566 (size=12493) 2024-11-26T15:27:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T15:27:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:54,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:54,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:54,750 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126debcf9d848f14b57b37d512245d9f510_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126debcf9d848f14b57b37d512245d9f510_ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:54,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/be1ded815fe049e2bb321468dbb5833c, store: [table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:54,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/be1ded815fe049e2bb321468dbb5833c is 175, key is test_row_0/A:col10/1732634872555/Put/seqid=0 2024-11-26T15:27:54,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742391_1567 (size=31105) 2024-11-26T15:27:54,754 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/be1ded815fe049e2bb321468dbb5833c 2024-11-26T15:27:54,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/ba6538b13f7342469657bb58650d180b is 50, key is test_row_0/B:col10/1732634872555/Put/seqid=0 2024-11-26T15:27:54,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742392_1568 (size=12151) 2024-11-26T15:27:54,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634934763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:54,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:54,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:54,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:54,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634934767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:54,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:54,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:54,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:54,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634934767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:54,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634934768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:54,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634934873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:54,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634934882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:54,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634934882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:54,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634934882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,016 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/984e150b47314eb3a254949277534611 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/984e150b47314eb3a254949277534611 2024-11-26T15:27:55,020 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ade03e3421a188de043c46d5d985046e/A of ade03e3421a188de043c46d5d985046e into 984e150b47314eb3a254949277534611(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:55,020 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:55,020 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/A, priority=13, startTime=1732634874192; duration=0sec 2024-11-26T15:27:55,020 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:55,020 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:A 2024-11-26T15:27:55,039 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/37546ae74735493fb7b19b8283bcefde as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/37546ae74735493fb7b19b8283bcefde 2024-11-26T15:27:55,042 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ade03e3421a188de043c46d5d985046e/C of ade03e3421a188de043c46d5d985046e into 37546ae74735493fb7b19b8283bcefde(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:55,042 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:55,042 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/C, priority=13, startTime=1732634874192; duration=0sec 2024-11-26T15:27:55,042 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:55,042 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:C 2024-11-26T15:27:55,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634935083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634935092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634935092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634935092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,163 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/ba6538b13f7342469657bb58650d180b 2024-11-26T15:27:55,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/305c2710e5f74b6590f50f514d7e923d is 50, key is test_row_0/C:col10/1732634872555/Put/seqid=0 2024-11-26T15:27:55,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742393_1569 (size=12151) 2024-11-26T15:27:55,172 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/305c2710e5f74b6590f50f514d7e923d 2024-11-26T15:27:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T15:27:55,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/be1ded815fe049e2bb321468dbb5833c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/be1ded815fe049e2bb321468dbb5833c 2024-11-26T15:27:55,177 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/be1ded815fe049e2bb321468dbb5833c, entries=150, sequenceid=168, filesize=30.4 K 2024-11-26T15:27:55,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/ba6538b13f7342469657bb58650d180b as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/ba6538b13f7342469657bb58650d180b 2024-11-26T15:27:55,181 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/ba6538b13f7342469657bb58650d180b, entries=150, sequenceid=168, filesize=11.9 K 2024-11-26T15:27:55,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/305c2710e5f74b6590f50f514d7e923d as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/305c2710e5f74b6590f50f514d7e923d 2024-11-26T15:27:55,184 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/305c2710e5f74b6590f50f514d7e923d, entries=150, sequenceid=168, filesize=11.9 K 2024-11-26T15:27:55,185 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for ade03e3421a188de043c46d5d985046e in 845ms, sequenceid=168, compaction requested=false 2024-11-26T15:27:55,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:55,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:55,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-26T15:27:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-26T15:27:55,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-26T15:27:55,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1510 sec 2024-11-26T15:27:55,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.1550 sec 2024-11-26T15:27:55,239 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T15:27:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:55,395 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:55,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ade03e3421a188de043c46d5d985046e 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-26T15:27:55,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a3090cf149ab47af914568355af6d754_ade03e3421a188de043c46d5d985046e is 50, key is test_row_0/A:col10/1732634874766/Put/seqid=0 2024-11-26T15:27:55,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742394_1570 (size=14794) 2024-11-26T15:27:55,404 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:55,407 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a3090cf149ab47af914568355af6d754_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a3090cf149ab47af914568355af6d754_ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:55,407 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/9ddddb1ba61c4f6fb44b463db4df21ed, store: [table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:55,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/9ddddb1ba61c4f6fb44b463db4df21ed is 175, key is test_row_0/A:col10/1732634874766/Put/seqid=0 2024-11-26T15:27:55,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742395_1571 (size=39749) 2024-11-26T15:27:55,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634935407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634935407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634935412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634935412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634935523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634935523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634935523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634935523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634935593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,595 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in ade03e3421a188de043c46d5d985046e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634935733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634935733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634935733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634935733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:55,811 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/9ddddb1ba61c4f6fb44b463db4df21ed 2024-11-26T15:27:55,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/356b93fedafe4c64a40ce1387066fc95 is 50, key is test_row_0/B:col10/1732634874766/Put/seqid=0 2024-11-26T15:27:55,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742396_1572 (size=12151) 2024-11-26T15:27:56,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634936042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:56,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634936042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:56,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634936042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:56,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634936043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T15:27:56,193 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:56,193 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:27:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-26T15:27:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T15:27:56,195 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:27:56,195 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:27:56,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:27:56,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/356b93fedafe4c64a40ce1387066fc95 2024-11-26T15:27:56,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/1f5a434dd9694bd0ad0dd5fdaff2f3fe is 50, key is test_row_0/C:col10/1732634874766/Put/seqid=0 2024-11-26T15:27:56,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742397_1573 (size=12151) 2024-11-26T15:27:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T15:27:56,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T15:27:56,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:56,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:56,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:56,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:56,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:56,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T15:27:56,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:56,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:56,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:56,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:56,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T15:27:56,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634936552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:56,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634936552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:56,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:56,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:56,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:56,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:56,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:56,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634936552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:56,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:56,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:56,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:56,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:56,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634936553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:56,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:56,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:56,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:56,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/1f5a434dd9694bd0ad0dd5fdaff2f3fe 2024-11-26T15:27:56,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/9ddddb1ba61c4f6fb44b463db4df21ed as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9ddddb1ba61c4f6fb44b463db4df21ed 2024-11-26T15:27:56,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9ddddb1ba61c4f6fb44b463db4df21ed, entries=200, sequenceid=198, filesize=38.8 K 2024-11-26T15:27:56,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/356b93fedafe4c64a40ce1387066fc95 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/356b93fedafe4c64a40ce1387066fc95 2024-11-26T15:27:56,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/356b93fedafe4c64a40ce1387066fc95, entries=150, sequenceid=198, filesize=11.9 K 2024-11-26T15:27:56,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/1f5a434dd9694bd0ad0dd5fdaff2f3fe as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/1f5a434dd9694bd0ad0dd5fdaff2f3fe 2024-11-26T15:27:56,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/1f5a434dd9694bd0ad0dd5fdaff2f3fe, entries=150, sequenceid=198, filesize=11.9 K 2024-11-26T15:27:56,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for ade03e3421a188de043c46d5d985046e in 1245ms, sequenceid=198, compaction requested=true 2024-11-26T15:27:56,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:56,641 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:56,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:27:56,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:56,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:27:56,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:56,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:27:56,641 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:56,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:56,641 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:56,641 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/A is initiating minor compaction (all files) 2024-11-26T15:27:56,641 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/A in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:56,642 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/984e150b47314eb3a254949277534611, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/be1ded815fe049e2bb321468dbb5833c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9ddddb1ba61c4f6fb44b463db4df21ed] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=99.9 K 2024-11-26T15:27:56,642 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:56,642 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/984e150b47314eb3a254949277534611, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/be1ded815fe049e2bb321468dbb5833c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9ddddb1ba61c4f6fb44b463db4df21ed] 2024-11-26T15:27:56,642 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:56,642 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/B is initiating minor compaction (all files) 2024-11-26T15:27:56,642 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/B in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:56,642 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/ea563d0af4294587a7f3d6e1180298ce, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/ba6538b13f7342469657bb58650d180b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/356b93fedafe4c64a40ce1387066fc95] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=35.9 K 2024-11-26T15:27:56,643 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 984e150b47314eb3a254949277534611, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732634871401 2024-11-26T15:27:56,643 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea563d0af4294587a7f3d6e1180298ce, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732634871401 2024-11-26T15:27:56,643 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting be1ded815fe049e2bb321468dbb5833c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732634872553 2024-11-26T15:27:56,643 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba6538b13f7342469657bb58650d180b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732634872553 2024-11-26T15:27:56,643 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ddddb1ba61c4f6fb44b463db4df21ed, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732634874766 2024-11-26T15:27:56,643 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 356b93fedafe4c64a40ce1387066fc95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732634874766 2024-11-26T15:27:56,649 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#B#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:56,649 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/458b06c9abda48dea3ae5b1a3fae6815 is 50, key is test_row_0/B:col10/1732634874766/Put/seqid=0 2024-11-26T15:27:56,650 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:56,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-26T15:27:56,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:56,652 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing ade03e3421a188de043c46d5d985046e 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-26T15:27:56,652 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411268028f635f51a4da9889950a4c654aac0_ade03e3421a188de043c46d5d985046e store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:56,654 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411268028f635f51a4da9889950a4c654aac0_ade03e3421a188de043c46d5d985046e, store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:56,654 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268028f635f51a4da9889950a4c654aac0_ade03e3421a188de043c46d5d985046e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:56,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742398_1574 (size=12595) 2024-11-26T15:27:56,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742399_1575 (size=4469) 2024-11-26T15:27:56,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b9b141f3ed4b480f81d29d7a59fec8d2_ade03e3421a188de043c46d5d985046e is 50, key is test_row_0/A:col10/1732634875410/Put/seqid=0 2024-11-26T15:27:56,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742400_1576 (size=12304) 2024-11-26T15:27:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T15:27:57,058 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#A#compaction#483 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:57,059 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/fb04fba53a2c4dc88c76f1377ec3de46 is 175, key is test_row_0/A:col10/1732634874766/Put/seqid=0 2024-11-26T15:27:57,059 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/458b06c9abda48dea3ae5b1a3fae6815 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/458b06c9abda48dea3ae5b1a3fae6815 2024-11-26T15:27:57,063 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ade03e3421a188de043c46d5d985046e/B of ade03e3421a188de043c46d5d985046e into 458b06c9abda48dea3ae5b1a3fae6815(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:57,063 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:57,063 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/B, priority=13, startTime=1732634876641; duration=0sec 2024-11-26T15:27:57,063 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:57,063 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:B 2024-11-26T15:27:57,063 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:57,064 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:57,064 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/C is initiating minor compaction (all files) 2024-11-26T15:27:57,064 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/C in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:57,064 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/37546ae74735493fb7b19b8283bcefde, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/305c2710e5f74b6590f50f514d7e923d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/1f5a434dd9694bd0ad0dd5fdaff2f3fe] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=35.9 K 2024-11-26T15:27:57,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:57,065 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37546ae74735493fb7b19b8283bcefde, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732634871401 2024-11-26T15:27:57,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742401_1577 (size=31549) 2024-11-26T15:27:57,067 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 305c2710e5f74b6590f50f514d7e923d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732634872553 2024-11-26T15:27:57,068 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f5a434dd9694bd0ad0dd5fdaff2f3fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732634874766 2024-11-26T15:27:57,069 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b9b141f3ed4b480f81d29d7a59fec8d2_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b9b141f3ed4b480f81d29d7a59fec8d2_ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:57,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/4ae9e6fc32e34ec78666589d6313ea47, store: [table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:57,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/4ae9e6fc32e34ec78666589d6313ea47 is 175, key is test_row_0/A:col10/1732634875410/Put/seqid=0 2024-11-26T15:27:57,073 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#C#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:57,073 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/15529cc183de4ff6b768d1bf37cfecbe is 50, key is test_row_0/C:col10/1732634874766/Put/seqid=0 2024-11-26T15:27:57,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742402_1578 (size=31105) 2024-11-26T15:27:57,075 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/4ae9e6fc32e34ec78666589d6313ea47 2024-11-26T15:27:57,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742403_1579 (size=12595) 2024-11-26T15:27:57,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/a012aa6762cd4717a821b7fb61294f62 is 50, key is test_row_0/B:col10/1732634875410/Put/seqid=0 2024-11-26T15:27:57,082 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/15529cc183de4ff6b768d1bf37cfecbe as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/15529cc183de4ff6b768d1bf37cfecbe 2024-11-26T15:27:57,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742404_1580 (size=12151) 2024-11-26T15:27:57,086 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ade03e3421a188de043c46d5d985046e/C of ade03e3421a188de043c46d5d985046e into 15529cc183de4ff6b768d1bf37cfecbe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:57,086 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:57,086 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/C, priority=13, startTime=1732634876641; duration=0sec 2024-11-26T15:27:57,086 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:57,086 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:C 2024-11-26T15:27:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T15:27:57,470 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/fb04fba53a2c4dc88c76f1377ec3de46 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/fb04fba53a2c4dc88c76f1377ec3de46 2024-11-26T15:27:57,473 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ade03e3421a188de043c46d5d985046e/A of ade03e3421a188de043c46d5d985046e into fb04fba53a2c4dc88c76f1377ec3de46(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:27:57,473 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:57,474 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/A, priority=13, startTime=1732634876640; duration=0sec 2024-11-26T15:27:57,474 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:57,474 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:A 2024-11-26T15:27:57,486 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/a012aa6762cd4717a821b7fb61294f62 2024-11-26T15:27:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/d4f112682d644423b6f6fda759b74e09 is 50, key is test_row_0/C:col10/1732634875410/Put/seqid=0 2024-11-26T15:27:57,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742405_1581 (size=12151) 2024-11-26T15:27:57,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:57,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634937591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:57,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:57,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634937592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:57,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634937592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:57,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634937593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:57,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:57,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:57,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:57,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634937702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:57,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:57,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634937702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:57,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:57,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:57,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634937702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:57,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634937702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:57,894 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/d4f112682d644423b6f6fda759b74e09 2024-11-26T15:27:57,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:57,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634937912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:57,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634937912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:57,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634937912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:57,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634937912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:57,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/4ae9e6fc32e34ec78666589d6313ea47 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/4ae9e6fc32e34ec78666589d6313ea47 2024-11-26T15:27:57,975 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/4ae9e6fc32e34ec78666589d6313ea47, entries=150, sequenceid=208, filesize=30.4 K 2024-11-26T15:27:57,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/a012aa6762cd4717a821b7fb61294f62 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/a012aa6762cd4717a821b7fb61294f62 2024-11-26T15:27:57,978 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/a012aa6762cd4717a821b7fb61294f62, entries=150, sequenceid=208, filesize=11.9 K 2024-11-26T15:27:57,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/d4f112682d644423b6f6fda759b74e09 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/d4f112682d644423b6f6fda759b74e09 2024-11-26T15:27:57,981 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/d4f112682d644423b6f6fda759b74e09, entries=150, sequenceid=208, filesize=11.9 K 2024-11-26T15:27:57,982 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for ade03e3421a188de043c46d5d985046e in 1331ms, sequenceid=208, compaction requested=false 2024-11-26T15:27:57,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:57,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:57,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-26T15:27:57,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-26T15:27:57,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-26T15:27:57,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7870 sec 2024-11-26T15:27:57,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.7920 sec 2024-11-26T15:27:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:58,225 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:27:58,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ade03e3421a188de043c46d5d985046e 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-26T15:27:58,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126af7966559a9f43aa83ead53231a7fc44_ade03e3421a188de043c46d5d985046e is 50, key is test_row_0/A:col10/1732634877586/Put/seqid=0 2024-11-26T15:27:58,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634938227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:58,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634938230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:58,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742406_1582 (size=14794) 2024-11-26T15:27:58,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634938231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634938231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:58,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T15:27:58,342 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:27:58,342 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-26T15:27:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-26T15:27:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T15:27:58,344 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T15:27:58,345 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T15:27:58,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:58,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T15:27:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634938342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:58,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634938342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634938342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634938342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T15:27:58,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T15:27:58,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:58,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:58,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:58,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:58,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:58,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:58,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634938552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:58,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634938552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:58,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634938552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:58,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634938552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:58,635 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:58,638 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126af7966559a9f43aa83ead53231a7fc44_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126af7966559a9f43aa83ead53231a7fc44_ade03e3421a188de043c46d5d985046e 2024-11-26T15:27:58,639 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/34583bd9060a4d8c8b706e5d4eccc970, store: [table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:58,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/34583bd9060a4d8c8b706e5d4eccc970 is 175, key is test_row_0/A:col10/1732634877586/Put/seqid=0 2024-11-26T15:27:58,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742407_1583 (size=39749) 2024-11-26T15:27:58,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T15:27:58,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:58,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:58,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:58,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:58,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:58,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T15:27:58,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T15:27:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:58,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:58,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634938863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:58,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:58,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634938863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:58,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:58,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634938863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:58,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:58,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634938863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:58,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:58,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:58,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T15:27:58,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:58,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:58,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:58,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:58,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T15:27:59,043 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/34583bd9060a4d8c8b706e5d4eccc970 2024-11-26T15:27:59,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/5cf0395d98304605b3b285997becb8f8 is 50, key is test_row_0/B:col10/1732634877586/Put/seqid=0 2024-11-26T15:27:59,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742408_1584 (size=12151) 2024-11-26T15:27:59,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T15:27:59,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:59,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:59,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:59,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T15:27:59,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:59,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:59,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:59,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634939372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:59,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:59,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:59,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:59,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634939372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:59,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:59,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:59,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:59,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634939372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:59,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634939372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:59,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T15:27:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:59,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/5cf0395d98304605b3b285997becb8f8 2024-11-26T15:27:59,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/00b7bde7976c448984ebb9cc94ba1061 is 50, key is test_row_0/C:col10/1732634877586/Put/seqid=0 2024-11-26T15:27:59,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742409_1585 (size=12151) 2024-11-26T15:27:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T15:27:59,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T15:27:59,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:59,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:59,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:59,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:27:59,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50614 deadline: 1732634939622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:27:59,625 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in ade03e3421a188de043c46d5d985046e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:27:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:27:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:27:59,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T15:27:59,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:59,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:59,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:59,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/00b7bde7976c448984ebb9cc94ba1061 2024-11-26T15:27:59,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/34583bd9060a4d8c8b706e5d4eccc970 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/34583bd9060a4d8c8b706e5d4eccc970 2024-11-26T15:27:59,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/34583bd9060a4d8c8b706e5d4eccc970, entries=200, sequenceid=238, filesize=38.8 K 2024-11-26T15:27:59,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/5cf0395d98304605b3b285997becb8f8 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/5cf0395d98304605b3b285997becb8f8 2024-11-26T15:27:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/5cf0395d98304605b3b285997becb8f8, entries=150, sequenceid=238, filesize=11.9 K 2024-11-26T15:27:59,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T15:27:59,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:59,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:27:59,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:59,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/00b7bde7976c448984ebb9cc94ba1061 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/00b7bde7976c448984ebb9cc94ba1061 2024-11-26T15:27:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:27:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/00b7bde7976c448984ebb9cc94ba1061, entries=150, sequenceid=238, filesize=11.9 K 2024-11-26T15:27:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for ade03e3421a188de043c46d5d985046e in 1648ms, sequenceid=238, compaction requested=true 2024-11-26T15:27:59,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:27:59,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:27:59,873 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:59,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,874 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:27:59,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:27:59,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:27:59,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:27:59,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:27:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,874 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:59,874 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:27:59,874 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/B is initiating minor compaction (all files) 2024-11-26T15:27:59,874 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/A is initiating minor compaction (all files) 2024-11-26T15:27:59,874 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/B in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:59,874 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/A in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:59,874 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/fb04fba53a2c4dc88c76f1377ec3de46, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/4ae9e6fc32e34ec78666589d6313ea47, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/34583bd9060a4d8c8b706e5d4eccc970] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=100.0 K 2024-11-26T15:27:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,874 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/458b06c9abda48dea3ae5b1a3fae6815, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/a012aa6762cd4717a821b7fb61294f62, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/5cf0395d98304605b3b285997becb8f8] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=36.0 K 2024-11-26T15:27:59,874 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:27:59,874 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/fb04fba53a2c4dc88c76f1377ec3de46, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/4ae9e6fc32e34ec78666589d6313ea47, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/34583bd9060a4d8c8b706e5d4eccc970] 2024-11-26T15:27:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,875 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb04fba53a2c4dc88c76f1377ec3de46, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732634874766 2024-11-26T15:27:59,875 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 458b06c9abda48dea3ae5b1a3fae6815, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732634874766 2024-11-26T15:27:59,875 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting a012aa6762cd4717a821b7fb61294f62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732634875403 2024-11-26T15:27:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,875 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf0395d98304605b3b285997becb8f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732634877586 2024-11-26T15:27:59,875 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ae9e6fc32e34ec78666589d6313ea47, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732634875403 2024-11-26T15:27:59,876 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34583bd9060a4d8c8b706e5d4eccc970, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732634877586 2024-11-26T15:27:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,881 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#B#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:27:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,881 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/36b56974f3254c618578a1c569bd880f is 50, key is test_row_0/B:col10/1732634877586/Put/seqid=0 2024-11-26T15:27:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,882 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,884 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126a7658d6fbfed4a5a8813b040d10a9fd0_ade03e3421a188de043c46d5d985046e store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742410_1586 (size=12697) 2024-11-26T15:27:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,886 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126a7658d6fbfed4a5a8813b040d10a9fd0_ade03e3421a188de043c46d5d985046e, store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,886 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a7658d6fbfed4a5a8813b040d10a9fd0_ade03e3421a188de043c46d5d985046e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:27:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742411_1587 (size=4469) 2024-11-26T15:27:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:27:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-26T15:28:00,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:28:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,022 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing ade03e3421a188de043c46d5d985046e 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-26T15:28:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267ea096cdb72140dbbbeee1c952708b4a_ade03e3421a188de043c46d5d985046e is 50, key is test_row_1/A:col10/1732634878230/Put/seqid=0 2024-11-26T15:28:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742412_1588 (size=9814) 2024-11-26T15:28:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,290 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/36b56974f3254c618578a1c569bd880f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/36b56974f3254c618578a1c569bd880f 2024-11-26T15:28:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,295 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ade03e3421a188de043c46d5d985046e/B of ade03e3421a188de043c46d5d985046e into 36b56974f3254c618578a1c569bd880f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:28:00,295 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:28:00,295 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/B, priority=13, startTime=1732634879874; duration=0sec 2024-11-26T15:28:00,295 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:28:00,295 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:B 2024-11-26T15:28:00,295 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:28:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,296 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:28:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,297 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/C is initiating minor compaction (all files) 2024-11-26T15:28:00,297 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/C in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:28:00,297 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/15529cc183de4ff6b768d1bf37cfecbe, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/d4f112682d644423b6f6fda759b74e09, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/00b7bde7976c448984ebb9cc94ba1061] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=36.0 K 2024-11-26T15:28:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,298 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 15529cc183de4ff6b768d1bf37cfecbe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732634874766 2024-11-26T15:28:00,298 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting d4f112682d644423b6f6fda759b74e09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732634875403 2024-11-26T15:28:00,298 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 00b7bde7976c448984ebb9cc94ba1061, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732634877586 2024-11-26T15:28:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,304 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:28:00,305 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/bf52fb959df64651ad67711f4bdbad72 is 50, key is test_row_0/C:col10/1732634877586/Put/seqid=0 2024-11-26T15:28:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,306 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#A#compaction#492 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:28:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,307 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/61d542e47c2f464ca1b79de5db2347bc is 175, key is test_row_0/A:col10/1732634877586/Put/seqid=0 2024-11-26T15:28:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742413_1589 (size=31651) 2024-11-26T15:28:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,347 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/61d542e47c2f464ca1b79de5db2347bc as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/61d542e47c2f464ca1b79de5db2347bc 2024-11-26T15:28:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,351 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ade03e3421a188de043c46d5d985046e/A of ade03e3421a188de043c46d5d985046e into 61d542e47c2f464ca1b79de5db2347bc(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:28:00,351 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:28:00,351 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/A, priority=13, startTime=1732634879873; duration=0sec 2024-11-26T15:28:00,351 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:28:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,351 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:A 2024-11-26T15:28:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742414_1590 (size=12697) 2024-11-26T15:28:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:00,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. as already flushing 2024-11-26T15:28:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634940416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:00,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:00,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634940419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:00,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634940420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634940420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:00,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:00,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:00,434 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267ea096cdb72140dbbbeee1c952708b4a_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267ea096cdb72140dbbbeee1c952708b4a_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:00,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/92044d3ec94b4ba6ab531c400f9d0461, store: [table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:28:00,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/92044d3ec94b4ba6ab531c400f9d0461 is 175, key is test_row_1/A:col10/1732634878230/Put/seqid=0 2024-11-26T15:28:00,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742415_1591 (size=22461) 2024-11-26T15:28:00,440 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/92044d3ec94b4ba6ab531c400f9d0461 2024-11-26T15:28:00,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/13994cdb75bd4ce5ab9f140419b9392c is 50, key is test_row_1/B:col10/1732634878230/Put/seqid=0 2024-11-26T15:28:00,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742416_1592 (size=9757) 2024-11-26T15:28:00,454 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/13994cdb75bd4ce5ab9f140419b9392c 2024-11-26T15:28:00,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/e74490de56ac40099d60e7cc4ae2977a is 50, key is test_row_1/C:col10/1732634878230/Put/seqid=0 2024-11-26T15:28:00,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742417_1593 (size=9757) 2024-11-26T15:28:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T15:28:00,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634940522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:00,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:00,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634940532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:00,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634940532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:00,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634940532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:00,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634940732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:00,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:00,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634940742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634940742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634940742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:00,762 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/bf52fb959df64651ad67711f4bdbad72 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/bf52fb959df64651ad67711f4bdbad72 2024-11-26T15:28:00,765 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ade03e3421a188de043c46d5d985046e/C of ade03e3421a188de043c46d5d985046e into bf52fb959df64651ad67711f4bdbad72(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:28:00,765 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:28:00,765 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/C, priority=13, startTime=1732634879874; duration=0sec 2024-11-26T15:28:00,765 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:28:00,765 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:C 2024-11-26T15:28:00,878 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/e74490de56ac40099d60e7cc4ae2977a 2024-11-26T15:28:00,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/92044d3ec94b4ba6ab531c400f9d0461 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/92044d3ec94b4ba6ab531c400f9d0461 2024-11-26T15:28:00,884 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/92044d3ec94b4ba6ab531c400f9d0461, entries=100, sequenceid=247, filesize=21.9 K 2024-11-26T15:28:00,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/13994cdb75bd4ce5ab9f140419b9392c as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/13994cdb75bd4ce5ab9f140419b9392c 2024-11-26T15:28:00,887 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/13994cdb75bd4ce5ab9f140419b9392c, entries=100, sequenceid=247, filesize=9.5 K 2024-11-26T15:28:00,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/e74490de56ac40099d60e7cc4ae2977a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e74490de56ac40099d60e7cc4ae2977a 2024-11-26T15:28:00,890 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e74490de56ac40099d60e7cc4ae2977a, entries=100, sequenceid=247, filesize=9.5 K 2024-11-26T15:28:00,890 INFO [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for ade03e3421a188de043c46d5d985046e in 868ms, sequenceid=247, compaction requested=false 2024-11-26T15:28:00,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:28:00,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:28:00,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-26T15:28:00,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-26T15:28:00,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-26T15:28:00,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5460 sec 2024-11-26T15:28:00,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.5500 sec 2024-11-26T15:28:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:01,045 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:28:01,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ade03e3421a188de043c46d5d985046e 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-26T15:28:01,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cdda089dc2df45f3b36f268288b93398_ade03e3421a188de043c46d5d985046e is 50, key is test_row_0/A:col10/1732634881044/Put/seqid=0 2024-11-26T15:28:01,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742418_1594 (size=14994) 2024-11-26T15:28:01,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:01,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634941053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634941053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:01,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634941055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634941056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:01,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:01,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634941163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634941163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:01,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:01,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:01,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634941372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:01,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:01,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634941372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:01,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:01,455 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:01,457 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cdda089dc2df45f3b36f268288b93398_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cdda089dc2df45f3b36f268288b93398_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:01,458 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/ac15d577d12b49428e1c27a108574d6f, store: [table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:28:01,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/ac15d577d12b49428e1c27a108574d6f is 175, key is test_row_0/A:col10/1732634881044/Put/seqid=0 2024-11-26T15:28:01,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742419_1595 (size=39949) 2024-11-26T15:28:01,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634941562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:01,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634941562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:01,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634941682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634941682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:01,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:01,862 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/ac15d577d12b49428e1c27a108574d6f 2024-11-26T15:28:01,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/0b5716ed392f4668ac0bbff0598bf5e8 is 50, key is test_row_0/B:col10/1732634881044/Put/seqid=0 2024-11-26T15:28:01,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742420_1596 (size=12301) 2024-11-26T15:28:02,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50584 deadline: 1732634942193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:02,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:02,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50616 deadline: 1732634942193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:02,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:02,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:02,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:02,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/0b5716ed392f4668ac0bbff0598bf5e8 2024-11-26T15:28:02,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/e9b73db1c6e941a8a1a15524261506b0 is 50, key is test_row_0/C:col10/1732634881044/Put/seqid=0 2024-11-26T15:28:02,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742421_1597 (size=12301) 2024-11-26T15:28:02,409 INFO [Thread-2357 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2357. 2024-11-26T15:28:02,409 DEBUG [Thread-2357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:28:02,409 DEBUG [Thread-2357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:02,409 INFO [Thread-2353 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2353. 2024-11-26T15:28:02,410 DEBUG [Thread-2357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:02,410 DEBUG [Thread-2353 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:28:02,410 DEBUG [Thread-2353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:02,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:28:02,410 DEBUG [Thread-2353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:02,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:28:02,410 INFO [Thread-2349 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2349. 2024-11-26T15:28:02,410 DEBUG [Thread-2349 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:28:02,410 DEBUG [Thread-2349 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:02,411 DEBUG [Thread-2349 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:02,411 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:28:02,411 INFO [Thread-2351 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2351. 2024-11-26T15:28:02,411 DEBUG [Thread-2351 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:28:02,411 DEBUG [Thread-2351 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:02,411 DEBUG [Thread-2351 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:02,412 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:28:02,412 INFO [Thread-2355 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2355. 2024-11-26T15:28:02,412 DEBUG [Thread-2355 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:28:02,412 DEBUG [Thread-2355 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:02,412 DEBUG [Thread-2355 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:02,412 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:28:02,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-26T15:28:02,514 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:28:02,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:02,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50600 deadline: 1732634942573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:02,576 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_1' in ade03e3421a188de043c46d5d985046e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:28:02,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:02,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:02,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:02,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T15:28:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50586 deadline: 1732634942583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:02,583 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6cc2bc8bf733,45729,1732634697146 for 'test_row_0' in ade03e3421a188de043c46d5d985046e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-26T15:28:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 , the old value is region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T15:28:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., hostname=6cc2bc8bf733,45729,1732634697146, seqNum=5 because the exception is null or not the one we care about 2024-11-26T15:28:02,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/e9b73db1c6e941a8a1a15524261506b0 2024-11-26T15:28:02,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/ac15d577d12b49428e1c27a108574d6f as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/ac15d577d12b49428e1c27a108574d6f 2024-11-26T15:28:02,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/ac15d577d12b49428e1c27a108574d6f, entries=200, sequenceid=279, filesize=39.0 K 2024-11-26T15:28:02,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/0b5716ed392f4668ac0bbff0598bf5e8 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0b5716ed392f4668ac0bbff0598bf5e8 2024-11-26T15:28:02,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0b5716ed392f4668ac0bbff0598bf5e8, entries=150, sequenceid=279, filesize=12.0 K 2024-11-26T15:28:02,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/e9b73db1c6e941a8a1a15524261506b0 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e9b73db1c6e941a8a1a15524261506b0 2024-11-26T15:28:02,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e9b73db1c6e941a8a1a15524261506b0, entries=150, sequenceid=279, filesize=12.0 K 2024-11-26T15:28:02,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for ade03e3421a188de043c46d5d985046e in 1654ms, sequenceid=279, compaction requested=true 2024-11-26T15:28:02,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:28:02,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T15:28:02,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:28:02,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T15:28:02,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:28:02,700 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:28:02,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ade03e3421a188de043c46d5d985046e:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T15:28:02,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:28:02,700 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:28:02,700 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:28:02,700 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:28:02,700 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/A is initiating minor compaction (all files) 2024-11-26T15:28:02,700 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/B is initiating minor compaction (all files) 2024-11-26T15:28:02,700 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/A in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:28:02,700 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/B in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:28:02,700 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/36b56974f3254c618578a1c569bd880f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/13994cdb75bd4ce5ab9f140419b9392c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0b5716ed392f4668ac0bbff0598bf5e8] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=33.9 K 2024-11-26T15:28:02,700 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/61d542e47c2f464ca1b79de5db2347bc, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/92044d3ec94b4ba6ab531c400f9d0461, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/ac15d577d12b49428e1c27a108574d6f] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=91.9 K 2024-11-26T15:28:02,701 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:28:02,701 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. files: [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/61d542e47c2f464ca1b79de5db2347bc, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/92044d3ec94b4ba6ab531c400f9d0461, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/ac15d577d12b49428e1c27a108574d6f] 2024-11-26T15:28:02,701 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b56974f3254c618578a1c569bd880f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732634877586 2024-11-26T15:28:02,701 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61d542e47c2f464ca1b79de5db2347bc, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732634877586 2024-11-26T15:28:02,701 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 13994cdb75bd4ce5ab9f140419b9392c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732634878229 2024-11-26T15:28:02,701 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92044d3ec94b4ba6ab531c400f9d0461, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732634878229 2024-11-26T15:28:02,701 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b5716ed392f4668ac0bbff0598bf5e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732634880418 2024-11-26T15:28:02,701 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac15d577d12b49428e1c27a108574d6f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732634880418 2024-11-26T15:28:02,706 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:28:02,707 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#B#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:28:02,707 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/9f078d6042214fadae2397d1d020e645 is 50, key is test_row_0/B:col10/1732634881044/Put/seqid=0 2024-11-26T15:28:02,707 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126460a46480c694fa89510172970bb7c39_ade03e3421a188de043c46d5d985046e store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:28:02,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742422_1598 (size=12949) 2024-11-26T15:28:02,711 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126460a46480c694fa89510172970bb7c39_ade03e3421a188de043c46d5d985046e, store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:28:02,711 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126460a46480c694fa89510172970bb7c39_ade03e3421a188de043c46d5d985046e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:28:02,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742423_1599 (size=4469) 2024-11-26T15:28:03,116 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#A#compaction#501 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:28:03,118 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/d1f1fcb966934fa48e0a190daec7d799 is 175, key is test_row_0/A:col10/1732634881044/Put/seqid=0 2024-11-26T15:28:03,121 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/9f078d6042214fadae2397d1d020e645 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/9f078d6042214fadae2397d1d020e645 2024-11-26T15:28:03,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742424_1600 (size=31903) 2024-11-26T15:28:03,126 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ade03e3421a188de043c46d5d985046e/B of ade03e3421a188de043c46d5d985046e into 9f078d6042214fadae2397d1d020e645(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:28:03,126 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:28:03,126 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/B, priority=13, startTime=1732634882699; duration=0sec 2024-11-26T15:28:03,126 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T15:28:03,126 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:B 2024-11-26T15:28:03,126 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T15:28:03,127 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T15:28:03,127 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1541): ade03e3421a188de043c46d5d985046e/C is initiating minor compaction (all files) 2024-11-26T15:28:03,127 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ade03e3421a188de043c46d5d985046e/C in TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:28:03,127 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/bf52fb959df64651ad67711f4bdbad72, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e74490de56ac40099d60e7cc4ae2977a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e9b73db1c6e941a8a1a15524261506b0] into tmpdir=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp, totalSize=33.9 K 2024-11-26T15:28:03,127 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting bf52fb959df64651ad67711f4bdbad72, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732634877586 2024-11-26T15:28:03,128 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting e74490de56ac40099d60e7cc4ae2977a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732634878229 2024-11-26T15:28:03,128 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] compactions.Compactor(225): Compacting e9b73db1c6e941a8a1a15524261506b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732634880418 2024-11-26T15:28:03,135 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ade03e3421a188de043c46d5d985046e#C#compaction#502 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:28:03,135 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/48747fc11d334792b4210ab310e9f4c4 is 50, key is test_row_0/C:col10/1732634881044/Put/seqid=0 2024-11-26T15:28:03,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742425_1601 (size=12949) 2024-11-26T15:28:03,205 INFO [Thread-2346 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2346. 2024-11-26T15:28:03,205 DEBUG [Thread-2346 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:28:03,205 DEBUG [Thread-2346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:03,205 DEBUG [Thread-2346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:03,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:28:03,216 INFO [Thread-2338 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2338. 2024-11-26T15:28:03,216 DEBUG [Thread-2338 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:28:03,216 DEBUG [Thread-2338 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:03,216 DEBUG [Thread-2338 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:03,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:28:03,535 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/d1f1fcb966934fa48e0a190daec7d799 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/d1f1fcb966934fa48e0a190daec7d799 2024-11-26T15:28:03,539 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ade03e3421a188de043c46d5d985046e/A of ade03e3421a188de043c46d5d985046e into d1f1fcb966934fa48e0a190daec7d799(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:28:03,539 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:28:03,539 INFO [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/A, priority=13, startTime=1732634882699; duration=0sec 2024-11-26T15:28:03,539 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:28:03,539 DEBUG [RS:0;6cc2bc8bf733:45729-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:A 2024-11-26T15:28:03,541 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/48747fc11d334792b4210ab310e9f4c4 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/48747fc11d334792b4210ab310e9f4c4 2024-11-26T15:28:03,544 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ade03e3421a188de043c46d5d985046e/C of ade03e3421a188de043c46d5d985046e into 48747fc11d334792b4210ab310e9f4c4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:28:03,544 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:28:03,544 INFO [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., storeName=ade03e3421a188de043c46d5d985046e/C, priority=13, startTime=1732634882700; duration=0sec 2024-11-26T15:28:03,544 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T15:28:03,544 DEBUG [RS:0;6cc2bc8bf733:45729-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ade03e3421a188de043c46d5d985046e:C 2024-11-26T15:28:04,598 INFO [Thread-2344 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2344. 2024-11-26T15:28:04,598 DEBUG [Thread-2344 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:28:04,598 DEBUG [Thread-2344 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:04,598 DEBUG [Thread-2344 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:04,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:28:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45729 {}] regionserver.HRegion(8855): Flush requested on ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:04,605 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T15:28:04,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ade03e3421a188de043c46d5d985046e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-26T15:28:04,606 INFO [Thread-2340 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2340. 2024-11-26T15:28:04,606 DEBUG [Thread-2340 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:28:04,606 DEBUG [Thread-2340 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:04,606 DEBUG [Thread-2340 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:04,606 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:28:04,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126be73c5935d12449c9a2d568bfe0ee26f_ade03e3421a188de043c46d5d985046e is 50, key is test_row_0/A:col10/1732634884604/Put/seqid=0 2024-11-26T15:28:04,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742426_1602 (size=12454) 2024-11-26T15:28:05,017 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:05,024 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126be73c5935d12449c9a2d568bfe0ee26f_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126be73c5935d12449c9a2d568bfe0ee26f_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:05,026 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/d4eb8a777f3048fb91b227ab253e7d55, store: [table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:28:05,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/d4eb8a777f3048fb91b227ab253e7d55 is 175, key is test_row_0/A:col10/1732634884604/Put/seqid=0 2024-11-26T15:28:05,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742427_1603 (size=31255) 2024-11-26T15:28:05,433 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/d4eb8a777f3048fb91b227ab253e7d55 2024-11-26T15:28:05,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/461b006812434125afa007412c2803fe is 50, key is test_row_0/B:col10/1732634884604/Put/seqid=0 2024-11-26T15:28:05,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742428_1604 (size=12301) 2024-11-26T15:28:05,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/461b006812434125afa007412c2803fe 2024-11-26T15:28:05,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/25a16c62a4f5466e8063924129d0584a is 50, key is test_row_0/C:col10/1732634884604/Put/seqid=0 2024-11-26T15:28:05,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742429_1605 (size=12301) 2024-11-26T15:28:05,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/25a16c62a4f5466e8063924129d0584a 2024-11-26T15:28:05,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/d4eb8a777f3048fb91b227ab253e7d55 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/d4eb8a777f3048fb91b227ab253e7d55 2024-11-26T15:28:05,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/d4eb8a777f3048fb91b227ab253e7d55, entries=150, sequenceid=293, filesize=30.5 K 2024-11-26T15:28:05,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/461b006812434125afa007412c2803fe as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/461b006812434125afa007412c2803fe 2024-11-26T15:28:05,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/461b006812434125afa007412c2803fe, entries=150, sequenceid=293, filesize=12.0 K 2024-11-26T15:28:05,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/25a16c62a4f5466e8063924129d0584a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/25a16c62a4f5466e8063924129d0584a 2024-11-26T15:28:05,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/25a16c62a4f5466e8063924129d0584a, entries=150, sequenceid=293, filesize=12.0 K 2024-11-26T15:28:05,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for ade03e3421a188de043c46d5d985046e in 1281ms, sequenceid=293, compaction requested=false 2024-11-26T15:28:05,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ade03e3421a188de043c46d5d985046e: 2024-11-26T15:28:09,724 INFO [Thread-2342 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2342. 2024-11-26T15:28:09,724 DEBUG [Thread-2342 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T15:28:09,724 DEBUG [Thread-2342 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:09,724 DEBUG [Thread-2342 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:09,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:28:09,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T15:28:09,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-26T15:28:09,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-26T15:28:09,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-26T15:28:09,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-26T15:28:09,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-26T15:28:09,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T15:28:09,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4399 2024-11-26T15:28:09,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4264 2024-11-26T15:28:09,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4203 2024-11-26T15:28:09,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4389 2024-11-26T15:28:09,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4251 2024-11-26T15:28:09,724 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T15:28:09,724 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T15:28:09,724 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T15:28:09,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:09,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:09,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:28:09,725 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-26T15:28:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T15:28:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T15:28:09,728 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634889727"}]},"ts":"1732634889727"} 2024-11-26T15:28:09,729 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T15:28:09,729 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T15:28:09,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T15:28:09,731 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ade03e3421a188de043c46d5d985046e, UNASSIGN}] 2024-11-26T15:28:09,732 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ade03e3421a188de043c46d5d985046e, UNASSIGN 2024-11-26T15:28:09,733 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=ade03e3421a188de043c46d5d985046e, regionState=CLOSING, regionLocation=6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:09,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ade03e3421a188de043c46d5d985046e, UNASSIGN because future has completed 2024-11-26T15:28:09,734 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37081 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=6cc2bc8bf733,45729,1732634697146, table=TestAcidGuarantees, region=ade03e3421a188de043c46d5d985046e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T15:28:09,734 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T15:28:09,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146}] 2024-11-26T15:28:09,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T15:28:09,888 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:09,888 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T15:28:09,889 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing ade03e3421a188de043c46d5d985046e, disabling compactions & flushes 2024-11-26T15:28:09,889 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:28:09,889 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:28:09,889 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. after waiting 0 ms 2024-11-26T15:28:09,889 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:28:09,889 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing ade03e3421a188de043c46d5d985046e 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-26T15:28:09,899 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265fa9c4671d78462196092f3873849ab5_ade03e3421a188de043c46d5d985046e is 50, key is test_row_1/A:col10/1732634889723/Put/seqid=0 2024-11-26T15:28:09,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742430_1606 (size=7374) 2024-11-26T15:28:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T15:28:10,305 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:10,313 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265fa9c4671d78462196092f3873849ab5_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265fa9c4671d78462196092f3873849ab5_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:10,314 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/a19e011613cc4a55b040a7b5da61718e, store: [table=TestAcidGuarantees family=A region=ade03e3421a188de043c46d5d985046e] 2024-11-26T15:28:10,315 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/a19e011613cc4a55b040a7b5da61718e is 175, key is test_row_1/A:col10/1732634889723/Put/seqid=0 2024-11-26T15:28:10,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742431_1607 (size=13865) 2024-11-26T15:28:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T15:28:10,721 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/a19e011613cc4a55b040a7b5da61718e 2024-11-26T15:28:10,733 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/fbe4cbe17c1845e78b5d5c9999038576 is 50, key is test_row_1/B:col10/1732634889723/Put/seqid=0 2024-11-26T15:28:10,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742432_1608 (size=7415) 2024-11-26T15:28:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T15:28:11,137 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/fbe4cbe17c1845e78b5d5c9999038576 2024-11-26T15:28:11,142 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/f63d544513994d67ac98744e37b336d3 is 50, key is test_row_1/C:col10/1732634889723/Put/seqid=0 2024-11-26T15:28:11,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742433_1609 (size=7415) 2024-11-26T15:28:11,545 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/f63d544513994d67ac98744e37b336d3 2024-11-26T15:28:11,549 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/A/a19e011613cc4a55b040a7b5da61718e as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/a19e011613cc4a55b040a7b5da61718e 2024-11-26T15:28:11,552 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/a19e011613cc4a55b040a7b5da61718e, entries=50, sequenceid=297, filesize=13.5 K 2024-11-26T15:28:11,553 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/B/fbe4cbe17c1845e78b5d5c9999038576 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/fbe4cbe17c1845e78b5d5c9999038576 2024-11-26T15:28:11,556 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/fbe4cbe17c1845e78b5d5c9999038576, entries=50, sequenceid=297, filesize=7.2 K 2024-11-26T15:28:11,556 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/.tmp/C/f63d544513994d67ac98744e37b336d3 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/f63d544513994d67ac98744e37b336d3 2024-11-26T15:28:11,560 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/f63d544513994d67ac98744e37b336d3, entries=50, sequenceid=297, filesize=7.2 K 2024-11-26T15:28:11,561 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for ade03e3421a188de043c46d5d985046e in 1672ms, sequenceid=297, compaction requested=true 2024-11-26T15:28:11,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/6b7aeb7120b346e1a3c8c699d247ac2e, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9873a12ff9dc4709a025ce79a0d2069d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9da57d4d15054f5b86bcdaf925674b62, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/dbe05ded48494db685a7d729dc99fc79, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/4605b4e8a15748039a3634308d462954, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/7fcdebd28bc8421e8f4ae3a482162c51, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/850f3bab17104b678c6fdb7dca8fc9bb, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9a8dded33d934c98a1c5e24acf020c4f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/3312b339ba5545658ea6c43c462c3dc3, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/984e150b47314eb3a254949277534611, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/37cd1b3c069040e1825f1a5034941c02, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/be1ded815fe049e2bb321468dbb5833c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9ddddb1ba61c4f6fb44b463db4df21ed, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/fb04fba53a2c4dc88c76f1377ec3de46, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/4ae9e6fc32e34ec78666589d6313ea47, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/34583bd9060a4d8c8b706e5d4eccc970, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/61d542e47c2f464ca1b79de5db2347bc, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/92044d3ec94b4ba6ab531c400f9d0461, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/ac15d577d12b49428e1c27a108574d6f] to archive 2024-11-26T15:28:11,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:28:11,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/6b7aeb7120b346e1a3c8c699d247ac2e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/6b7aeb7120b346e1a3c8c699d247ac2e 2024-11-26T15:28:11,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9873a12ff9dc4709a025ce79a0d2069d to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9873a12ff9dc4709a025ce79a0d2069d 2024-11-26T15:28:11,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9da57d4d15054f5b86bcdaf925674b62 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9da57d4d15054f5b86bcdaf925674b62 2024-11-26T15:28:11,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/dbe05ded48494db685a7d729dc99fc79 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/dbe05ded48494db685a7d729dc99fc79 2024-11-26T15:28:11,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/4605b4e8a15748039a3634308d462954 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/4605b4e8a15748039a3634308d462954 2024-11-26T15:28:11,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/7fcdebd28bc8421e8f4ae3a482162c51 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/7fcdebd28bc8421e8f4ae3a482162c51 2024-11-26T15:28:11,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/850f3bab17104b678c6fdb7dca8fc9bb to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/850f3bab17104b678c6fdb7dca8fc9bb 2024-11-26T15:28:11,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9a8dded33d934c98a1c5e24acf020c4f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9a8dded33d934c98a1c5e24acf020c4f 2024-11-26T15:28:11,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/3312b339ba5545658ea6c43c462c3dc3 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/3312b339ba5545658ea6c43c462c3dc3 2024-11-26T15:28:11,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/984e150b47314eb3a254949277534611 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/984e150b47314eb3a254949277534611 2024-11-26T15:28:11,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/37cd1b3c069040e1825f1a5034941c02 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/37cd1b3c069040e1825f1a5034941c02 2024-11-26T15:28:11,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/be1ded815fe049e2bb321468dbb5833c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/be1ded815fe049e2bb321468dbb5833c 2024-11-26T15:28:11,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9ddddb1ba61c4f6fb44b463db4df21ed to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/9ddddb1ba61c4f6fb44b463db4df21ed 2024-11-26T15:28:11,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/fb04fba53a2c4dc88c76f1377ec3de46 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/fb04fba53a2c4dc88c76f1377ec3de46 2024-11-26T15:28:11,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/4ae9e6fc32e34ec78666589d6313ea47 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/4ae9e6fc32e34ec78666589d6313ea47 2024-11-26T15:28:11,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/34583bd9060a4d8c8b706e5d4eccc970 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/34583bd9060a4d8c8b706e5d4eccc970 2024-11-26T15:28:11,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/61d542e47c2f464ca1b79de5db2347bc to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/61d542e47c2f464ca1b79de5db2347bc 2024-11-26T15:28:11,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/92044d3ec94b4ba6ab531c400f9d0461 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/92044d3ec94b4ba6ab531c400f9d0461 2024-11-26T15:28:11,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/ac15d577d12b49428e1c27a108574d6f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/ac15d577d12b49428e1c27a108574d6f 2024-11-26T15:28:11,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/9a38040a30f44ae4b898ab90b98b589d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/9f45ff24c47342cda0b07fdcf10730f4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/b22d82be3c3b429e96f8ee1d272a2b39, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/8d76f18527ef4a0d9cfbc621f9998732, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/bdce483fcf97457699cc4f08dcb4fe2b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/80124ca6db5640148f14a05217768ed7, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/aec443b8542a47e3b4860078b50a059c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0eb15181927c4bfc9d32b4c2c86202ab, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0e583294dc024c6e86724614e6d3d71c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/ea563d0af4294587a7f3d6e1180298ce, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/cb602e0b99de46bb910ada95ba3d9b1b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/ba6538b13f7342469657bb58650d180b, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/458b06c9abda48dea3ae5b1a3fae6815, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/356b93fedafe4c64a40ce1387066fc95, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/a012aa6762cd4717a821b7fb61294f62, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/36b56974f3254c618578a1c569bd880f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/5cf0395d98304605b3b285997becb8f8, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/13994cdb75bd4ce5ab9f140419b9392c, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0b5716ed392f4668ac0bbff0598bf5e8] to archive 2024-11-26T15:28:11,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:28:11,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/9a38040a30f44ae4b898ab90b98b589d to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/9a38040a30f44ae4b898ab90b98b589d 2024-11-26T15:28:11,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/9f45ff24c47342cda0b07fdcf10730f4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/9f45ff24c47342cda0b07fdcf10730f4 2024-11-26T15:28:11,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/b22d82be3c3b429e96f8ee1d272a2b39 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/b22d82be3c3b429e96f8ee1d272a2b39 2024-11-26T15:28:11,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/8d76f18527ef4a0d9cfbc621f9998732 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/8d76f18527ef4a0d9cfbc621f9998732 2024-11-26T15:28:11,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/bdce483fcf97457699cc4f08dcb4fe2b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/bdce483fcf97457699cc4f08dcb4fe2b 2024-11-26T15:28:11,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/80124ca6db5640148f14a05217768ed7 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/80124ca6db5640148f14a05217768ed7 2024-11-26T15:28:11,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/aec443b8542a47e3b4860078b50a059c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/aec443b8542a47e3b4860078b50a059c 2024-11-26T15:28:11,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0eb15181927c4bfc9d32b4c2c86202ab to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0eb15181927c4bfc9d32b4c2c86202ab 2024-11-26T15:28:11,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0e583294dc024c6e86724614e6d3d71c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0e583294dc024c6e86724614e6d3d71c 2024-11-26T15:28:11,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/ea563d0af4294587a7f3d6e1180298ce to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/ea563d0af4294587a7f3d6e1180298ce 2024-11-26T15:28:11,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/cb602e0b99de46bb910ada95ba3d9b1b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/cb602e0b99de46bb910ada95ba3d9b1b 2024-11-26T15:28:11,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/ba6538b13f7342469657bb58650d180b to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/ba6538b13f7342469657bb58650d180b 2024-11-26T15:28:11,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/458b06c9abda48dea3ae5b1a3fae6815 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/458b06c9abda48dea3ae5b1a3fae6815 2024-11-26T15:28:11,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/356b93fedafe4c64a40ce1387066fc95 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/356b93fedafe4c64a40ce1387066fc95 2024-11-26T15:28:11,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/a012aa6762cd4717a821b7fb61294f62 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/a012aa6762cd4717a821b7fb61294f62 2024-11-26T15:28:11,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/36b56974f3254c618578a1c569bd880f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/36b56974f3254c618578a1c569bd880f 2024-11-26T15:28:11,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/5cf0395d98304605b3b285997becb8f8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/5cf0395d98304605b3b285997becb8f8 2024-11-26T15:28:11,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/13994cdb75bd4ce5ab9f140419b9392c to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/13994cdb75bd4ce5ab9f140419b9392c 2024-11-26T15:28:11,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0b5716ed392f4668ac0bbff0598bf5e8 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/0b5716ed392f4668ac0bbff0598bf5e8 2024-11-26T15:28:11,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/5c68ac94ccfb4336aae28fa31867887f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/c7a77910bc59469f9bb0a01b4b2369d9, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/3007539562524785b4ceccb75fe80caf, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/017e32b4f99440a291eda6c3146241a5, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/1a21b727c7cf4dfc89a6746c25a51043, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/bad99cef66524af0bf97464ed84c063d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/bd9a33b2c07b48f789c5f356446f7ea2, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e9fea665a9494cb0baeddb17bb0a6d0f, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/3e47c8fb85584815aca05da73cba3034, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/37546ae74735493fb7b19b8283bcefde, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/c557f72a12854a238eefcbaa12c2dec4, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/305c2710e5f74b6590f50f514d7e923d, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/15529cc183de4ff6b768d1bf37cfecbe, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/1f5a434dd9694bd0ad0dd5fdaff2f3fe, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/d4f112682d644423b6f6fda759b74e09, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/bf52fb959df64651ad67711f4bdbad72, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/00b7bde7976c448984ebb9cc94ba1061, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e74490de56ac40099d60e7cc4ae2977a, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e9b73db1c6e941a8a1a15524261506b0] to archive 2024-11-26T15:28:11,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:28:11,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/5c68ac94ccfb4336aae28fa31867887f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/5c68ac94ccfb4336aae28fa31867887f 2024-11-26T15:28:11,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/c7a77910bc59469f9bb0a01b4b2369d9 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/c7a77910bc59469f9bb0a01b4b2369d9 2024-11-26T15:28:11,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/3007539562524785b4ceccb75fe80caf to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/3007539562524785b4ceccb75fe80caf 2024-11-26T15:28:11,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/017e32b4f99440a291eda6c3146241a5 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/017e32b4f99440a291eda6c3146241a5 2024-11-26T15:28:11,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/1a21b727c7cf4dfc89a6746c25a51043 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/1a21b727c7cf4dfc89a6746c25a51043 2024-11-26T15:28:11,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/bad99cef66524af0bf97464ed84c063d to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/bad99cef66524af0bf97464ed84c063d 2024-11-26T15:28:11,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/bd9a33b2c07b48f789c5f356446f7ea2 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/bd9a33b2c07b48f789c5f356446f7ea2 2024-11-26T15:28:11,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e9fea665a9494cb0baeddb17bb0a6d0f to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e9fea665a9494cb0baeddb17bb0a6d0f 2024-11-26T15:28:11,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/3e47c8fb85584815aca05da73cba3034 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/3e47c8fb85584815aca05da73cba3034 2024-11-26T15:28:11,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/37546ae74735493fb7b19b8283bcefde to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/37546ae74735493fb7b19b8283bcefde 2024-11-26T15:28:11,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/c557f72a12854a238eefcbaa12c2dec4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/c557f72a12854a238eefcbaa12c2dec4 2024-11-26T15:28:11,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/305c2710e5f74b6590f50f514d7e923d to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/305c2710e5f74b6590f50f514d7e923d 2024-11-26T15:28:11,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/15529cc183de4ff6b768d1bf37cfecbe to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/15529cc183de4ff6b768d1bf37cfecbe 2024-11-26T15:28:11,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/1f5a434dd9694bd0ad0dd5fdaff2f3fe to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/1f5a434dd9694bd0ad0dd5fdaff2f3fe 2024-11-26T15:28:11,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/d4f112682d644423b6f6fda759b74e09 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/d4f112682d644423b6f6fda759b74e09 2024-11-26T15:28:11,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/bf52fb959df64651ad67711f4bdbad72 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/bf52fb959df64651ad67711f4bdbad72 2024-11-26T15:28:11,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/00b7bde7976c448984ebb9cc94ba1061 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/00b7bde7976c448984ebb9cc94ba1061 2024-11-26T15:28:11,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e74490de56ac40099d60e7cc4ae2977a to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e74490de56ac40099d60e7cc4ae2977a 2024-11-26T15:28:11,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e9b73db1c6e941a8a1a15524261506b0 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/e9b73db1c6e941a8a1a15524261506b0 2024-11-26T15:28:11,621 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/recovered.edits/300.seqid, newMaxSeqId=300, maxSeqId=4 2024-11-26T15:28:11,621 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. 2024-11-26T15:28:11,621 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for ade03e3421a188de043c46d5d985046e: Waiting for close lock at 1732634889888Running coprocessor pre-close hooks at 1732634889888Disabling compacts and flushes for region at 1732634889888Disabling writes for close at 1732634889889 (+1 ms)Obtaining lock to block concurrent updates at 1732634889889Preparing flush snapshotting stores in ade03e3421a188de043c46d5d985046e at 1732634889889Finished memstore snapshotting TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1732634889890 (+1 ms)Flushing stores of TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e. at 1732634889891 (+1 ms)Flushing ade03e3421a188de043c46d5d985046e/A: creating writer at 1732634889891Flushing mob file ade03e3421a188de043c46d5d985046e/A: appending metadata at 1732634889899 (+8 ms)Flushing mob file ade03e3421a188de043c46d5d985046e/A: closing flushed file at 1732634889899Flushing ade03e3421a188de043c46d5d985046e/A: appending metadata at 1732634890314 (+415 ms)Flushing ade03e3421a188de043c46d5d985046e/A: closing flushed file at 1732634890314Flushing ade03e3421a188de043c46d5d985046e/B: creating writer at 1732634890730 (+416 ms)Flushing ade03e3421a188de043c46d5d985046e/B: appending metadata at 1732634890732 (+2 ms)Flushing ade03e3421a188de043c46d5d985046e/B: closing flushed file at 1732634890732Flushing ade03e3421a188de043c46d5d985046e/C: creating writer at 1732634891140 (+408 ms)Flushing ade03e3421a188de043c46d5d985046e/C: appending metadata at 1732634891141 (+1 ms)Flushing ade03e3421a188de043c46d5d985046e/C: closing flushed file at 1732634891142 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44160c43: reopening flushed file at 1732634891548 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12a630d4: reopening flushed file at 1732634891552 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d9381c3: reopening flushed file at 1732634891556 (+4 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for ade03e3421a188de043c46d5d985046e in 1672ms, sequenceid=297, compaction requested=true at 1732634891561 (+5 ms)Writing region close event to WAL at 1732634891618 (+57 ms)Running coprocessor post-close hooks at 1732634891621 (+3 ms)Closed at 1732634891621 2024-11-26T15:28:11,623 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,623 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=ade03e3421a188de043c46d5d985046e, regionState=CLOSED 2024-11-26T15:28:11,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 because future has completed 2024-11-26T15:28:11,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-26T15:28:11,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure ade03e3421a188de043c46d5d985046e, server=6cc2bc8bf733,45729,1732634697146 in 1.8910 sec 2024-11-26T15:28:11,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-26T15:28:11,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ade03e3421a188de043c46d5d985046e, UNASSIGN in 1.8950 sec 2024-11-26T15:28:11,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-26T15:28:11,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8970 sec 2024-11-26T15:28:11,630 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634891629"}]},"ts":"1732634891629"} 2024-11-26T15:28:11,631 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T15:28:11,631 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T15:28:11,632 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9060 sec 2024-11-26T15:28:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T15:28:11,872 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:28:11,873 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-26T15:28:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:28:11,875 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:28:11,876 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:28:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-26T15:28:11,878 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,880 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C, FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/recovered.edits] 2024-11-26T15:28:11,882 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/a19e011613cc4a55b040a7b5da61718e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/a19e011613cc4a55b040a7b5da61718e 2024-11-26T15:28:11,883 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/d1f1fcb966934fa48e0a190daec7d799 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/d1f1fcb966934fa48e0a190daec7d799 2024-11-26T15:28:11,884 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/d4eb8a777f3048fb91b227ab253e7d55 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/A/d4eb8a777f3048fb91b227ab253e7d55 2024-11-26T15:28:11,886 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/461b006812434125afa007412c2803fe to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/461b006812434125afa007412c2803fe 2024-11-26T15:28:11,887 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/9f078d6042214fadae2397d1d020e645 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/9f078d6042214fadae2397d1d020e645 2024-11-26T15:28:11,887 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/fbe4cbe17c1845e78b5d5c9999038576 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/B/fbe4cbe17c1845e78b5d5c9999038576 2024-11-26T15:28:11,889 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/25a16c62a4f5466e8063924129d0584a to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/25a16c62a4f5466e8063924129d0584a 2024-11-26T15:28:11,890 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/48747fc11d334792b4210ab310e9f4c4 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/48747fc11d334792b4210ab310e9f4c4 2024-11-26T15:28:11,891 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/f63d544513994d67ac98744e37b336d3 to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/C/f63d544513994d67ac98744e37b336d3 2024-11-26T15:28:11,893 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/recovered.edits/300.seqid to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e/recovered.edits/300.seqid 2024-11-26T15:28:11,894 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/default/TestAcidGuarantees/ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,894 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T15:28:11,894 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T15:28:11,895 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T15:28:11,897 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261cdc74972acd469b9135492a13fa3142_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261cdc74972acd469b9135492a13fa3142_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,898 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112643b56d7b5b614483b8b100e15748fba8_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112643b56d7b5b614483b8b100e15748fba8_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,899 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264de566f68ce84c628a98e1978c48f152_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264de566f68ce84c628a98e1978c48f152_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,900 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126536cd76fd8224094b92d1b0c725acf07_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126536cd76fd8224094b92d1b0c725acf07_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,901 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126594df66c086b4874bd2c99c495e3a6a0_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126594df66c086b4874bd2c99c495e3a6a0_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,902 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265fa9c4671d78462196092f3873849ab5_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265fa9c4671d78462196092f3873849ab5_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,903 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267ea096cdb72140dbbbeee1c952708b4a_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267ea096cdb72140dbbbeee1c952708b4a_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,904 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a3090cf149ab47af914568355af6d754_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a3090cf149ab47af914568355af6d754_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,906 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126af7966559a9f43aa83ead53231a7fc44_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126af7966559a9f43aa83ead53231a7fc44_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,907 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b9b141f3ed4b480f81d29d7a59fec8d2_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b9b141f3ed4b480f81d29d7a59fec8d2_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,908 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126be73c5935d12449c9a2d568bfe0ee26f_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126be73c5935d12449c9a2d568bfe0ee26f_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,909 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c32dff3caacb4484a13281e1f5275d8e_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c32dff3caacb4484a13281e1f5275d8e_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,910 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c6a634a41cd24741a3d95c9bce5680a0_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c6a634a41cd24741a3d95c9bce5680a0_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,911 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cdda089dc2df45f3b36f268288b93398_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cdda089dc2df45f3b36f268288b93398_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,912 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cee64ffd523f4f60a9de9f7b26293a23_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cee64ffd523f4f60a9de9f7b26293a23_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,913 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126debcf9d848f14b57b37d512245d9f510_ade03e3421a188de043c46d5d985046e to hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126debcf9d848f14b57b37d512245d9f510_ade03e3421a188de043c46d5d985046e 2024-11-26T15:28:11,913 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T15:28:11,915 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:28:11,918 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T15:28:11,920 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T15:28:11,921 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:28:11,921 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T15:28:11,921 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732634891921"}]},"ts":"9223372036854775807"} 2024-11-26T15:28:11,923 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T15:28:11,923 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ade03e3421a188de043c46d5d985046e, NAME => 'TestAcidGuarantees,,1732634859278.ade03e3421a188de043c46d5d985046e.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T15:28:11,923 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T15:28:11,923 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732634891923"}]},"ts":"9223372036854775807"} 2024-11-26T15:28:11,925 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T15:28:11,926 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T15:28:11,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-26T15:28:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37081 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-26T15:28:11,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T15:28:11,984 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T15:28:11,997 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238 (was 237) - Thread LEAK? -, OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=346 (was 376), ProcessCount=11 (was 11), AvailableMemoryMB=9240 (was 9256) 2024-11-26T15:28:11,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T15:28:11,997 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T15:28:11,997 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T15:28:11,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:11,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:11,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:28:11,998 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T15:28:11,998 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1336910459, stopped=false 2024-11-26T15:28:11,998 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6cc2bc8bf733,37081,1732634696319 2024-11-26T15:28:12,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T15:28:12,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T15:28:12,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:12,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:12,088 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T15:28:12,089 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T15:28:12,089 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T15:28:12,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:12,090 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T15:28:12,090 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6cc2bc8bf733,45729,1732634697146' ***** 2024-11-26T15:28:12,090 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T15:28:12,090 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T15:28:12,090 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T15:28:12,091 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T15:28:12,091 INFO [RS:0;6cc2bc8bf733:45729 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T15:28:12,091 INFO [RS:0;6cc2bc8bf733:45729 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T15:28:12,091 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(959): stopping server 6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:12,091 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T15:28:12,091 INFO [RS:0;6cc2bc8bf733:45729 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6cc2bc8bf733:45729. 2024-11-26T15:28:12,091 DEBUG [RS:0;6cc2bc8bf733:45729 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T15:28:12,091 DEBUG [RS:0;6cc2bc8bf733:45729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:12,092 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T15:28:12,092 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T15:28:12,092 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T15:28:12,092 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T15:28:12,093 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T15:28:12,093 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-26T15:28:12,093 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T15:28:12,093 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T15:28:12,093 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T15:28:12,093 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T15:28:12,093 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T15:28:12,093 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T15:28:12,094 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-26T15:28:12,115 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/.tmp/info/038b544a8f9045d8b1d032b94d8a07ed is 95, key is TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175./info:/1732634728481/DeleteFamily/seqid=0 2024-11-26T15:28:12,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742434_1610 (size=6514) 2024-11-26T15:28:12,147 INFO [regionserver/6cc2bc8bf733:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T15:28:12,283 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=6cc2bc8bf733:37081 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 10 more 2024-11-26T15:28:12,294 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T15:28:12,494 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T15:28:12,519 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/.tmp/info/038b544a8f9045d8b1d032b94d8a07ed 2024-11-26T15:28:12,548 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/.tmp/ns/1514abdba17c47ff8aa09ea80190c326 is 93, key is TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175./ns:/1732634728434/DeleteFamily/seqid=0 2024-11-26T15:28:12,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742435_1611 (size=5906) 2024-11-26T15:28:12,574 INFO [regionserver/6cc2bc8bf733:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T15:28:12,574 INFO [regionserver/6cc2bc8bf733:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T15:28:12,694 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T15:28:12,894 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T15:28:12,953 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/.tmp/ns/1514abdba17c47ff8aa09ea80190c326 2024-11-26T15:28:12,980 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/.tmp/rep_barrier/62bcdc9c68604c6e8abddc73bef0dda4 is 102, key is TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175./rep_barrier:/1732634728434/DeleteFamily/seqid=0 2024-11-26T15:28:12,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742436_1612 (size=6025) 2024-11-26T15:28:13,095 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T15:28:13,295 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T15:28:13,385 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/.tmp/rep_barrier/62bcdc9c68604c6e8abddc73bef0dda4 2024-11-26T15:28:13,411 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/.tmp/table/f22c6bc225834ba68505f8fdb3da1601 is 96, key is TestAcidGuarantees,,1732634701171.4c75b81af65e8ac430a5b8a152e75175./table:/1732634728434/DeleteFamily/seqid=0 2024-11-26T15:28:13,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742437_1613 (size=5936) 2024-11-26T15:28:13,495 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T15:28:13,696 DEBUG [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T15:28:13,816 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/.tmp/table/f22c6bc225834ba68505f8fdb3da1601 2024-11-26T15:28:13,826 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/.tmp/info/038b544a8f9045d8b1d032b94d8a07ed as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/info/038b544a8f9045d8b1d032b94d8a07ed 2024-11-26T15:28:13,830 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/info/038b544a8f9045d8b1d032b94d8a07ed, entries=12, sequenceid=90, filesize=6.4 K 2024-11-26T15:28:13,831 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/.tmp/ns/1514abdba17c47ff8aa09ea80190c326 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/ns/1514abdba17c47ff8aa09ea80190c326 2024-11-26T15:28:13,833 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/ns/1514abdba17c47ff8aa09ea80190c326, entries=8, sequenceid=90, filesize=5.8 K 2024-11-26T15:28:13,834 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/.tmp/rep_barrier/62bcdc9c68604c6e8abddc73bef0dda4 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/rep_barrier/62bcdc9c68604c6e8abddc73bef0dda4 2024-11-26T15:28:13,836 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/rep_barrier/62bcdc9c68604c6e8abddc73bef0dda4, entries=6, sequenceid=90, filesize=5.9 K 2024-11-26T15:28:13,836 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/.tmp/table/f22c6bc225834ba68505f8fdb3da1601 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/table/f22c6bc225834ba68505f8fdb3da1601 2024-11-26T15:28:13,838 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/table/f22c6bc225834ba68505f8fdb3da1601, entries=7, sequenceid=90, filesize=5.8 K 2024-11-26T15:28:13,839 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1746ms, sequenceid=90, compaction requested=false 2024-11-26T15:28:13,842 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-26T15:28:13,843 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T15:28:13,843 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T15:28:13,843 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732634892093Running coprocessor pre-close hooks at 1732634892093Disabling compacts and flushes for region at 1732634892093Disabling writes for close at 1732634892093Obtaining lock to block concurrent updates at 1732634892094 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732634892094Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732634892094Flushing stores of hbase:meta,,1.1588230740 at 1732634892094Flushing 1588230740/info: creating writer at 1732634892095 (+1 ms)Flushing 1588230740/info: appending metadata at 1732634892114 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732634892114Flushing 1588230740/ns: creating writer at 1732634892529 (+415 ms)Flushing 1588230740/ns: appending metadata at 1732634892548 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732634892548Flushing 1588230740/rep_barrier: creating writer at 1732634892963 (+415 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732634892980 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732634892980Flushing 1588230740/table: creating writer at 1732634893393 (+413 ms)Flushing 1588230740/table: appending metadata at 1732634893410 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732634893410Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@592885f3: reopening flushed file at 1732634893824 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25d8e14: reopening flushed file at 1732634893830 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b33a102: reopening flushed file at 1732634893833 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38a012a5: reopening flushed file at 1732634893836 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1746ms, sequenceid=90, compaction requested=false at 1732634893839 (+3 ms)Writing region close event to WAL at 1732634893840 (+1 ms)Running coprocessor post-close hooks at 1732634893843 (+3 ms)Closed at 1732634893843 2024-11-26T15:28:13,843 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T15:28:13,896 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(976): stopping server 6cc2bc8bf733,45729,1732634697146; all regions closed. 2024-11-26T15:28:13,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741834_1010 (size=25344) 2024-11-26T15:28:13,908 DEBUG [RS:0;6cc2bc8bf733:45729 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/oldWALs 2024-11-26T15:28:13,909 INFO [RS:0;6cc2bc8bf733:45729 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6cc2bc8bf733%2C45729%2C1732634697146.meta:.meta(num 1732634700702) 2024-11-26T15:28:13,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741832_1008 (size=14052494) 2024-11-26T15:28:13,914 DEBUG [RS:0;6cc2bc8bf733:45729 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/oldWALs 2024-11-26T15:28:13,914 INFO [RS:0;6cc2bc8bf733:45729 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6cc2bc8bf733%2C45729%2C1732634697146:(num 1732634699725) 2024-11-26T15:28:13,914 DEBUG [RS:0;6cc2bc8bf733:45729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:28:13,914 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T15:28:13,914 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T15:28:13,914 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.ChoreService(370): Chore service for: regionserver/6cc2bc8bf733:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T15:28:13,914 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T15:28:13,915 INFO [regionserver/6cc2bc8bf733:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T15:28:13,915 INFO [RS:0;6cc2bc8bf733:45729 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45729 2024-11-26T15:28:14,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6cc2bc8bf733,45729,1732634697146 2024-11-26T15:28:14,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T15:28:14,000 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T15:28:14,001 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6cc2bc8bf733,45729,1732634697146] 2024-11-26T15:28:14,011 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6cc2bc8bf733,45729,1732634697146 already deleted, retry=false 2024-11-26T15:28:14,011 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6cc2bc8bf733,45729,1732634697146 expired; onlineServers=0 2024-11-26T15:28:14,011 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6cc2bc8bf733,37081,1732634696319' ***** 2024-11-26T15:28:14,011 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T15:28:14,012 INFO [M:0;6cc2bc8bf733:37081 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T15:28:14,012 INFO [M:0;6cc2bc8bf733:37081 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T15:28:14,012 DEBUG [M:0;6cc2bc8bf733:37081 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T15:28:14,012 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T15:28:14,012 DEBUG [M:0;6cc2bc8bf733:37081 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T15:28:14,012 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster-HFileCleaner.small.0-1732634699361 {}] cleaner.HFileCleaner(306): Exit Thread[master/6cc2bc8bf733:0:becomeActiveMaster-HFileCleaner.small.0-1732634699361,5,FailOnTimeoutGroup] 2024-11-26T15:28:14,012 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster-HFileCleaner.large.0-1732634699360 {}] cleaner.HFileCleaner(306): Exit Thread[master/6cc2bc8bf733:0:becomeActiveMaster-HFileCleaner.large.0-1732634699360,5,FailOnTimeoutGroup] 2024-11-26T15:28:14,012 INFO [M:0;6cc2bc8bf733:37081 {}] hbase.ChoreService(370): Chore service for: master/6cc2bc8bf733:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T15:28:14,012 INFO [M:0;6cc2bc8bf733:37081 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T15:28:14,012 DEBUG [M:0;6cc2bc8bf733:37081 {}] master.HMaster(1795): Stopping service threads 2024-11-26T15:28:14,013 INFO [M:0;6cc2bc8bf733:37081 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T15:28:14,013 INFO [M:0;6cc2bc8bf733:37081 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T15:28:14,013 ERROR [M:0;6cc2bc8bf733:37081 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1087708870) connection to localhost/127.0.0.1:44107 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:44107,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-26T15:28:14,014 INFO [M:0;6cc2bc8bf733:37081 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T15:28:14,014 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T15:28:14,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T15:28:14,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:14,025 DEBUG [M:0;6cc2bc8bf733:37081 {}] zookeeper.ZKUtil(347): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T15:28:14,025 WARN [M:0;6cc2bc8bf733:37081 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T15:28:14,026 INFO [M:0;6cc2bc8bf733:37081 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/.lastflushedseqids 2024-11-26T15:28:14,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742438_1614 (size=72) 2024-11-26T15:28:14,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T15:28:14,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45729-0x10177bf52480001, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T15:28:14,112 INFO [RS:0;6cc2bc8bf733:45729 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T15:28:14,113 INFO [RS:0;6cc2bc8bf733:45729 {}] regionserver.HRegionServer(1031): Exiting; stopping=6cc2bc8bf733,45729,1732634697146; zookeeper connection closed. 2024-11-26T15:28:14,113 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@be496f2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@be496f2 2024-11-26T15:28:14,114 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-26T15:28:14,440 INFO [M:0;6cc2bc8bf733:37081 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T15:28:14,441 INFO [M:0;6cc2bc8bf733:37081 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T15:28:14,441 DEBUG [M:0;6cc2bc8bf733:37081 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T15:28:14,441 INFO [M:0;6cc2bc8bf733:37081 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T15:28:14,441 DEBUG [M:0;6cc2bc8bf733:37081 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T15:28:14,441 DEBUG [M:0;6cc2bc8bf733:37081 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T15:28:14,441 DEBUG [M:0;6cc2bc8bf733:37081 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T15:28:14,442 INFO [M:0;6cc2bc8bf733:37081 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=758.54 KB heapSize=932.87 KB 2024-11-26T15:28:14,465 DEBUG [M:0;6cc2bc8bf733:37081 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/86c275e9197949abb9e35ae346cf0a7a is 82, key is hbase:meta,,1/info:regioninfo/1732634700789/Put/seqid=0 2024-11-26T15:28:14,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742439_1615 (size=5672) 2024-11-26T15:28:14,870 INFO [M:0;6cc2bc8bf733:37081 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2155 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/86c275e9197949abb9e35ae346cf0a7a 2024-11-26T15:28:14,924 DEBUG [M:0;6cc2bc8bf733:37081 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/de80816fc9c04b87a381ad8827b601fa is 2276, key is \x00\x00\x00\x00\x00\x00\x00Z/proc:d/1732634795985/Put/seqid=0 2024-11-26T15:28:14,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742440_1616 (size=43501) 2024-11-26T15:28:15,330 INFO [M:0;6cc2bc8bf733:37081 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=757.98 KB at sequenceid=2155 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/de80816fc9c04b87a381ad8827b601fa 2024-11-26T15:28:15,337 INFO [M:0;6cc2bc8bf733:37081 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for de80816fc9c04b87a381ad8827b601fa 2024-11-26T15:28:15,356 DEBUG [M:0;6cc2bc8bf733:37081 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2eca97efa67f4718b93af89e472bb645 is 69, key is 6cc2bc8bf733,45729,1732634697146/rs:state/1732634699461/Put/seqid=0 2024-11-26T15:28:15,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073742441_1617 (size=5156) 2024-11-26T15:28:15,760 INFO [M:0;6cc2bc8bf733:37081 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2155 (bloomFilter=true), to=hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2eca97efa67f4718b93af89e472bb645 2024-11-26T15:28:15,765 DEBUG [M:0;6cc2bc8bf733:37081 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/86c275e9197949abb9e35ae346cf0a7a as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/86c275e9197949abb9e35ae346cf0a7a 2024-11-26T15:28:15,769 INFO [M:0;6cc2bc8bf733:37081 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/86c275e9197949abb9e35ae346cf0a7a, entries=8, sequenceid=2155, filesize=5.5 K 2024-11-26T15:28:15,769 DEBUG [M:0;6cc2bc8bf733:37081 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/de80816fc9c04b87a381ad8827b601fa as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/de80816fc9c04b87a381ad8827b601fa 2024-11-26T15:28:15,772 INFO [M:0;6cc2bc8bf733:37081 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for de80816fc9c04b87a381ad8827b601fa 2024-11-26T15:28:15,772 INFO [M:0;6cc2bc8bf733:37081 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/de80816fc9c04b87a381ad8827b601fa, entries=166, sequenceid=2155, filesize=42.5 K 2024-11-26T15:28:15,773 DEBUG [M:0;6cc2bc8bf733:37081 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2eca97efa67f4718b93af89e472bb645 as hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2eca97efa67f4718b93af89e472bb645 2024-11-26T15:28:15,775 INFO [M:0;6cc2bc8bf733:37081 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44107/user/jenkins/test-data/8f6a940d-32f5-ff25-f292-a1b9959923e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2eca97efa67f4718b93af89e472bb645, entries=1, sequenceid=2155, filesize=5.0 K 2024-11-26T15:28:15,776 INFO [M:0;6cc2bc8bf733:37081 {}] regionserver.HRegion(3140): Finished flush of dataSize ~758.54 KB/776741, heapSize ~932.57 KB/954952, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1335ms, sequenceid=2155, compaction requested=false 2024-11-26T15:28:15,777 INFO [M:0;6cc2bc8bf733:37081 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T15:28:15,777 DEBUG [M:0;6cc2bc8bf733:37081 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732634894441Disabling compacts and flushes for region at 1732634894441Disabling writes for close at 1732634894441Obtaining lock to block concurrent updates at 1732634894442 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732634894442Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=776741, getHeapSize=955192, getOffHeapSize=0, getCellsCount=2346 at 1732634894443 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732634894444 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732634894444Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732634894465 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732634894465Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732634894903 (+438 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732634894924 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732634894924Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732634895337 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732634895355 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732634895355Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@334c5f37: reopening flushed file at 1732634895764 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@711c0973: reopening flushed file at 1732634895769 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fe5b8d4: reopening flushed file at 1732634895772 (+3 ms)Finished flush of dataSize ~758.54 KB/776741, heapSize ~932.57 KB/954952, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1335ms, sequenceid=2155, compaction requested=false at 1732634895776 (+4 ms)Writing region close event to WAL at 1732634895777 (+1 ms)Closed at 1732634895777 2024-11-26T15:28:15,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40785 is added to blk_1073741830_1006 (size=917999) 2024-11-26T15:28:15,780 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T15:28:15,780 INFO [M:0;6cc2bc8bf733:37081 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T15:28:15,780 INFO [M:0;6cc2bc8bf733:37081 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37081 2024-11-26T15:28:15,781 INFO [M:0;6cc2bc8bf733:37081 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T15:28:15,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T15:28:15,946 INFO [M:0;6cc2bc8bf733:37081 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T15:28:15,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37081-0x10177bf52480000, quorum=127.0.0.1:51747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T15:28:15,990 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d1d0948{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T15:28:15,994 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@49b49063{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T15:28:15,994 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T15:28:15,994 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@58f8c8ee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T15:28:15,994 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6aa6ac55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/hadoop.log.dir/,STOPPED} 2024-11-26T15:28:15,997 WARN [BP-1143902897-172.17.0.3-1732634689990 heartbeating to localhost/127.0.0.1:44107 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T15:28:15,997 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T15:28:15,997 WARN [BP-1143902897-172.17.0.3-1732634689990 heartbeating to localhost/127.0.0.1:44107 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1143902897-172.17.0.3-1732634689990 (Datanode Uuid 257274e4-7b31-4604-b9d8-d69272e12d3c) service to localhost/127.0.0.1:44107 2024-11-26T15:28:15,997 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T15:28:16,001 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/cluster_1ef45550-199a-e5e8-727d-dd604c50436f/data/data1/current/BP-1143902897-172.17.0.3-1732634689990 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T15:28:16,001 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/cluster_1ef45550-199a-e5e8-727d-dd604c50436f/data/data2/current/BP-1143902897-172.17.0.3-1732634689990 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T15:28:16,002 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T15:28:16,009 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T15:28:16,010 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T15:28:16,010 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T15:28:16,010 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T15:28:16,010 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b9d34009-8298-eddd-8ded-e8161e9620cc/hadoop.log.dir/,STOPPED} 2024-11-26T15:28:16,024 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T15:28:16,150 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down